main.py - master - AlexMalch/Book_NoteBook

main.py
2 495 строк110 KB
AlexanderMalach
14 май 2026, 11:23
14 май 2026, 11:23da1151e
import csv
import json
import hashlib
import sqlite3
import time
import re
import shutil
import sys
import uuid
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

import fitz  # PyMuPDF
import numpy as np
from docx import Document
from openai import OpenAI
from openpyxl import load_workbook
from pypdf import PdfReader

from PySide6.QtCore import Qt, QThread, Signal
from PySide6.QtWidgets import (
    QApplication,
    QFileDialog,
    QFormLayout,
    QGroupBox,
    QHBoxLayout,
    QInputDialog,
    QLabel,
    QLineEdit,
    QListWidget,
    QListWidgetItem,
    QMainWindow,
    QMessageBox,
    QPushButton,
    QComboBox,
    QDoubleSpinBox,
    QSpinBox,
    QSplitter,
    QTextEdit,
    QVBoxLayout,
    QWidget,
)

APP_DIR = Path(__file__).parent
DATA_DIR = APP_DIR / "data"
PROJECTS_DIR = DATA_DIR / "projects"
SUPPORTED_EXTENSIONS = {".txt", ".md", ".pdf", ".docx", ".xlsx", ".csv"}
MAX_TABLE_ROWS_PER_SHEET = 2000
MAX_OVERVIEW_CHUNKS = 35
MAX_COMPARE_CHUNKS_PER_FILE = 14
MAX_AUTO_SUMMARY_RECORDS_PER_FILE = 18
MAX_AUTO_SUMMARY_CONTEXT_CHARS = 12000


# =========================
# Utils
# =========================

def ensure_dirs() -> None:
    PROJECTS_DIR.mkdir(parents=True, exist_ok=True)


def slugify(text: str) -> str:
    text = text.strip().lower()
    text = re.sub(r"[^\wа-яА-ЯёЁ]+", "_", text, flags=re.UNICODE)
    text = re.sub(r"_+", "_", text).strip("_")
    return text or "project"


def read_text_file(path: Path) -> str:
    for encoding in ("utf-8", "utf-8-sig", "cp1251", "latin-requirements1.txt"):
        try:
            return path.read_text(encoding=encoding)
        except UnicodeDecodeError:
            continue
    return path.read_text(errors="ignore")


def read_csv_text(path: Path) -> str:
    for encoding in ("utf-8", "utf-8-sig", "cp1251", "latin-requirements1.txt"):
        try:
            return path.read_text(encoding=encoding)
        except UnicodeDecodeError:
            continue
    return path.read_text(errors="ignore")


def split_long_piece_by_words(piece: str, max_chars: int) -> List[str]:
    """
    Крайний fallback: режем слишком длинный фрагмент по словам, а не по символам.
    Слово разрезается только если одно отдельное слово длиннее max_chars.
    """
    piece = clean_one_line(piece)
    if not piece:
        return []

    words = piece.split()
    parts: List[str] = []
    current_words: List[str] = []
    current_len = 0

    for word in words:
        extra = len(word) + (1 if current_words else 0)

        if current_words and current_len + extra > max_chars:
            parts.append(" ".join(current_words).strip())
            current_words = []
            current_len = 0

        if len(word) > max_chars:
            if current_words:
                parts.append(" ".join(current_words).strip())
                current_words = []
                current_len = 0
            for i in range(0, len(word), max_chars):
                parts.append(word[i:i + max_chars])
            continue

        current_words.append(word)
        current_len += extra

    if current_words:
        parts.append(" ".join(current_words).strip())

    return [p for p in parts if p]


def split_long_paragraph(paragraph: str, max_chars: int) -> List[str]:
    """
    Длинный абзац делим сначала по предложениям, потом по словам.
    """
    paragraph = paragraph.strip()
    if not paragraph:
        return []

    sentence_candidates = re.split(r"(?<=[.!?…;:])\s+|\n+", paragraph)
    sentence_candidates = [s.strip() for s in sentence_candidates if s.strip()]

    if not sentence_candidates:
        return split_long_piece_by_words(paragraph, max_chars)

    parts: List[str] = []
    current = ""

    for sentence in sentence_candidates:
        if len(sentence) > max_chars:
            if current.strip():
                parts.append(current.strip())
                current = ""
            parts.extend(split_long_piece_by_words(sentence, max_chars))
            continue

        if not current:
            current = sentence
        elif len(current) + 1 + len(sentence) <= max_chars:
            current += " " + sentence
        else:
            parts.append(current.strip())
            current = sentence

    if current.strip():
        parts.append(current.strip())

    return parts


def overlap_from_previous(text: str, overlap_chars: int) -> str:
    """
    Overlap делается по словам, а не тупым срезом строки.
    Поэтому новый chunk не начинается с половины слова.
    """
    text = clean_one_line(text)
    if not text or overlap_chars <= 0:
        return ""

    words = text.split()
    selected: List[str] = []
    total = 0

    for word in reversed(words):
        extra = len(word) + (1 if selected else 0)
        if selected and total + extra > overlap_chars:
            break
        selected.append(word)
        total += extra

    return " ".join(reversed(selected)).strip()


def chunk_text(text: str, max_chars: int = 1800, overlap: int = 250) -> List[str]:
    """
    Умная нарезка текста:
    requirements1.txt. Абзацы.
    2. Если абзац длинный — предложения.
    3. Если предложение длинное — слова.
    4. Символы только для экстремально длинных отдельных слов/URL.
    """
    max_chars = max(400, int(max_chars))
    overlap = max(0, min(int(overlap), max_chars // 2))

    text = re.sub(r"\r\n", "\n", text)
    text = re.sub(r"\n{3,}", "\n\n", text).strip()
    if not text:
        return []

    raw_paragraphs = re.split(r"\n\s*\n", text)
    pieces: List[str] = []

    for para in raw_paragraphs:
        para = para.strip()
        if not para:
            continue
        if len(para) <= max_chars:
            pieces.append(para)
        else:
            pieces.extend(split_long_paragraph(para, max_chars))

    chunks: List[str] = []
    current = ""

    for piece in pieces:
        piece = piece.strip()
        if not piece:
            continue

        separator = "\n\n"

        if not current:
            current = piece
        elif len(current) + len(separator) + len(piece) <= max_chars:
            current += separator + piece
        else:
            if current.strip():
                chunks.append(current.strip())
            overlap_text = overlap_from_previous(current, overlap)
            if overlap_text and len(overlap_text) + 2 + len(piece) <= max_chars:
                current = overlap_text + "\n\n" + piece
            else:
                current = piece

    if current.strip():
        chunks.append(current.strip())

    return chunks


def chunk_statistics(chunks: List[Any]) -> Dict[str, Any]:
    if not chunks:
        return {
            "count": 0,
            "avg_len": 0,
            "min_len": 0,
            "max_len": 0,
            "short_count": 0,
            "long_count": 0,
        }

    lengths = [len(chunk.text) for chunk in chunks]
    return {
        "count": len(chunks),
        "avg_len": sum(lengths) / len(lengths),
        "min_len": min(lengths),
        "max_len": max(lengths),
        "short_count": sum(1 for n in lengths if n < 120),
        "long_count": sum(1 for n in lengths if n > 3500),
    }


def cosine_similarity(a: List[float], b: List[float]) -> float:
    av = np.array(a, dtype=np.float32)
    bv = np.array(b, dtype=np.float32)
    denom = np.linalg.norm(av) * np.linalg.norm(bv)
    if denom == 0:
        return 0.0
    return float(np.dot(av, bv) / denom)


def clean_one_line(text: str) -> str:
    return re.sub(r"\s+", " ", text).strip()


def make_snippet(text: str, limit: int = 900) -> str:
    text = text.strip()
    if len(text) <= limit:
        return text
    return text[:limit].rstrip() + "..."


def confidence_label(score: Optional[float]) -> str:
    if score is None:
        return "нет данных"
    if score >= 0.75:
        return "высокая"
    if score >= 0.60:
        return "средняя"
    if score >= 0.45:
        return "слабая"
    return "очень слабая"


def safe_cell(value: Any) -> str:
    if value is None:
        return ""
    if isinstance(value, float):
        return f"{value:g}"
    return str(value).strip()


def row_to_text(headers: List[str], values: List[Any]) -> str:
    parts = []
    for i, value in enumerate(values):
        value_text = safe_cell(value)
        if not value_text:
            continue
        header = headers[i] if i < len(headers) and headers[i] else f"Колонка {i + 1}"
        parts.append(f"{header}: {value_text}")
    return "; ".join(parts)


# =========================
# Data models
# =========================

@dataclass
class Chunk:
    id: str
    source_file: str
    source_path: str
    page: Optional[int]
    text: str
    embedding: List[float]
    doc_type: str = "text"
    sheet: Optional[str] = None
    row: Optional[int] = None
    section: Optional[str] = None


@dataclass
class Project:
    id: str
    name: str
    path: str


# =========================
# Storage
# =========================


class ProjectStore:
    """
    SQLite-хранилище проекта.

    В версии 0.5 данные проекта больше не держатся в россыпи chunks.jsonl / summaries.json.
    Каждый проект получает собственную базу:
        data/projects/<project_id>/project.db

    project.json пока оставлен только как лёгкий файл-метка для списка проектов.
    """

    def __init__(self):
        ensure_dirs()

    def create_project(self, name: str) -> Project:
        project_id = f"{slugify(name)}_{uuid.uuid4().hex[:8]}"
        project_path = PROJECTS_DIR / project_id
        (project_path / "docs").mkdir(parents=True, exist_ok=True)
        project = Project(id=project_id, name=name.strip(), path=str(project_path))
        self._save_project(project)
        self.init_project_db(project)
        self.save_index_report(project, [])
        self.set_setting(project, "overall_summary", "")
        return project

    def list_projects(self) -> List[Project]:
        projects: List[Project] = []
        if not PROJECTS_DIR.exists():
            return projects
        for item in PROJECTS_DIR.iterdir():
            meta_path = item / "project.json"
            if meta_path.exists():
                try:
                    data = json.loads(meta_path.read_text(encoding="utf-8"))
                    project = Project(**data)
                    self.init_project_db(project)
                    projects.append(project)
                except Exception:
                    pass
        return sorted(projects, key=lambda p: p.name.lower())

    def _save_project(self, project: Project) -> None:
        path = Path(project.path)
        path.mkdir(parents=True, exist_ok=True)
        (path / "docs").mkdir(parents=True, exist_ok=True)
        (path / "project.json").write_text(
            json.dumps(asdict(project), ensure_ascii=False, indent=2),
            encoding="utf-8"
        )

    def db_path(self, project: Project) -> Path:
        return Path(project.path) / "project.db"

    def _connect(self, project: Project) -> sqlite3.Connection:
        self.init_project_db(project)
        conn = sqlite3.connect(str(self.db_path(project)))
        conn.row_factory = sqlite3.Row
        conn.execute("PRAGMA foreign_keys = ON")
        return conn

    def init_project_db(self, project: Project) -> None:
        path = Path(project.path)
        path.mkdir(parents=True, exist_ok=True)
        (path / "docs").mkdir(parents=True, exist_ok=True)
        db_path = self.db_path(project)
        conn = sqlite3.connect(str(db_path))
        try:
            conn.execute("PRAGMA foreign_keys = ON")
            conn.execute("""
                CREATE TABLE IF NOT EXISTS documents (
                    id TEXT PRIMARY KEY,
                    filename TEXT NOT NULL UNIQUE,
                    rel_path TEXT NOT NULL,
                    ext TEXT NOT NULL,
                    enabled INTEGER NOT NULL DEFAULT requirements1.txt,
                    file_hash TEXT,
                    size_bytes INTEGER,
                    modified_at REAL,
                    added_at REAL NOT NULL,
                    indexed_at REAL,
                    status TEXT NOT NULL DEFAULT 'new',
                    chunk_count INTEGER NOT NULL DEFAULT 0,
                    loader_report TEXT,
                    summary TEXT,
                    summary_updated_at REAL
                )
            """)
            conn.execute("""
                CREATE TABLE IF NOT EXISTS chunks (
                    id TEXT PRIMARY KEY,
                    document_id TEXT NOT NULL,
                    source_file TEXT NOT NULL,
                    source_path TEXT NOT NULL,
                    doc_type TEXT NOT NULL,
                    page INTEGER,
                    sheet TEXT,
                    row_number INTEGER,
                    section TEXT,
                    text TEXT NOT NULL,
                    embedding_json TEXT NOT NULL,
                    char_count INTEGER NOT NULL,
                    created_at REAL NOT NULL,
                    FOREIGN KEY(document_id) REFERENCES documents(id) ON DELETE CASCADE
                )
            """)
            conn.execute("""
                CREATE TABLE IF NOT EXISTS settings (
                    key TEXT PRIMARY KEY,
                    value TEXT
                )
            """)
            conn.execute("""
                CREATE TABLE IF NOT EXISTS index_report (
                    position INTEGER PRIMARY KEY,
                    line TEXT NOT NULL
                )
            """)
            conn.execute("CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)")
            conn.execute("CREATE INDEX IF NOT EXISTS idx_chunks_source_file ON chunks(source_file)")
            conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_enabled ON documents(enabled)")
            conn.commit()
        finally:
            conn.close()

    def file_hash(self, file_path: Path) -> str:
        h = hashlib.sha256()
        with file_path.open("rb") as f:
            for block in iter(lambda: f.read(1024 * 1024), b""):
                h.update(block)
        return h.hexdigest()

    def file_meta(self, file_path: Path) -> Dict[str, Any]:
        st = file_path.stat()
        return {
            "size_bytes": int(st.st_size),
            "modified_at": float(st.st_mtime),
            "file_hash": self.file_hash(file_path),
        }

    def set_setting(self, project: Project, key: str, value: str) -> None:
        with self._connect(project) as conn:
            conn.execute(
                "INSERT INTO settings(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
                (key, value),
            )
            conn.commit()

    def get_setting(self, project: Project, key: str, default: str = "") -> str:
        with self._connect(project) as conn:
            row = conn.execute("SELECT value FROM settings WHERE key=?", (key,)).fetchone()
            return str(row["value"]) if row and row["value"] is not None else default

    def get_index_signature(self, embedding_model: str, chunk_max_chars: int, chunk_overlap: int) -> str:
        return json.dumps(
            {
                "embedding_model": embedding_model,
                "chunk_max_chars": chunk_max_chars,
                "chunk_overlap": chunk_overlap,
                "schema": "sqlite_v1",
            },
            ensure_ascii=False,
            sort_keys=True,
        )

    def ensure_index_signature(self, project: Project, signature: str) -> bool:
        """
        Возвращает True, если настройки индекса изменились и старые chunks были сброшены.
        """
        old = self.get_setting(project, "index_signature", "")
        if old == signature:
            return False
        with self._connect(project) as conn:
            conn.execute("DELETE FROM chunks")
            conn.execute("UPDATE documents SET indexed_at=NULL, status='pending', chunk_count=0")
            conn.execute(
                "INSERT INTO settings(key, value) VALUES('index_signature', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
                (signature,),
            )
            conn.commit()
        return True

    def sync_document_state(self, project: Project) -> Dict[str, bool]:
        docs_dir = Path(project.path) / "docs"
        docs_dir.mkdir(parents=True, exist_ok=True)
        current_files = {
            p.name: p for p in docs_dir.iterdir()
            if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS
        }
        now = time.time()
        with self._connect(project) as conn:
            rows = conn.execute("SELECT filename FROM documents").fetchall()
            known = {row["filename"] for row in rows}

            # Файлы удалили руками из папки — убираем их из БД и chunks.
            for filename in sorted(known - set(current_files.keys())):
                conn.execute("DELETE FROM documents WHERE filename=?", (filename,))

            # Новые файлы в папке — добавляем как активные.
            for filename, path in current_files.items():
                rel_path = str(Path("docs") / filename)
                ext = path.suffix.lower()
                if filename not in known:
                    conn.execute(
                        """
                        INSERT INTO documents(id, filename, rel_path, ext, enabled, added_at, status)
                        VALUES(?, ?, ?, ?, requirements1.txt, ?, 'new')
                        """,
                        (uuid.uuid4().hex, filename, rel_path, ext, now),
                    )
                else:
                    conn.execute(
                        "UPDATE documents SET rel_path=?, ext=? WHERE filename=?",
                        (rel_path, ext, filename),
                    )
            conn.commit()

            rows = conn.execute("SELECT filename, enabled FROM documents ORDER BY lower(filename)").fetchall()
            return {row["filename"]: bool(row["enabled"]) for row in rows}

    def set_document_active(self, project: Project, filename: str, active: bool) -> None:
        self.sync_document_state(project)
        with self._connect(project) as conn:
            conn.execute("UPDATE documents SET enabled=? WHERE filename=?", (1 if active else 0, filename))
            conn.commit()

    def active_document_names(self, project: Project) -> set:
        self.sync_document_state(project)
        with self._connect(project) as conn:
            rows = conn.execute("SELECT filename FROM documents WHERE enabled=requirements1.txt").fetchall()
            return {row["filename"] for row in rows}

    def delete_project(self, project: Project) -> None:
        project_path = Path(project.path)
        if project_path.exists():
            shutil.rmtree(project_path)

    def delete_documents(self, project: Project, filenames: List[str]) -> None:
        docs_dir = Path(project.path) / "docs"
        for filename in filenames:
            target = docs_dir / filename
            if target.exists() and target.is_file():
                target.unlink()
        with self._connect(project) as conn:
            for filename in filenames:
                conn.execute("DELETE FROM documents WHERE filename=?", (filename,))
            conn.commit()
        self.sync_document_state(project)

    def import_files(self, project: Project, file_paths: List[str]) -> List[Path]:
        docs_dir = Path(project.path) / "docs"
        docs_dir.mkdir(parents=True, exist_ok=True)
        copied: List[Path] = []
        self.sync_document_state(project)
        now = time.time()
        with self._connect(project) as conn:
            for file_path in file_paths:
                src = Path(file_path)
                if not src.exists() or src.suffix.lower() not in SUPPORTED_EXTENSIONS:
                    continue
                dst = docs_dir / src.name
                if dst.exists():
                    dst = docs_dir / f"{dst.stem}_{uuid.uuid4().hex[:6]}{dst.suffix}"
                shutil.copy2(src, dst)
                copied.append(dst)
                conn.execute(
                    """
                    INSERT INTO documents(id, filename, rel_path, ext, enabled, added_at, status)
                    VALUES(?, ?, ?, ?, requirements1.txt, ?, 'new')
                    ON CONFLICT(filename) DO UPDATE SET
                        rel_path=excluded.rel_path,
                        ext=excluded.ext,
                        enabled=requirements1.txt,
                        status='pending'
                    """,
                    (uuid.uuid4().hex, dst.name, str(Path("docs") / dst.name), dst.suffix.lower(), now),
                )
            conn.commit()
        return copied

    def list_document_files(self, project: Project, active_only: bool = False) -> List[Path]:
        self.sync_document_state(project)
        docs_dir = Path(project.path) / "docs"
        with self._connect(project) as conn:
            if active_only:
                rows = conn.execute("SELECT filename FROM documents WHERE enabled=requirements1.txt ORDER BY lower(filename)").fetchall()
            else:
                rows = conn.execute("SELECT filename FROM documents ORDER BY lower(filename)").fetchall()
        files = []
        for row in rows:
            p = docs_dir / row["filename"]
            if p.exists() and p.is_file():
                files.append(p)
        return files

    def get_document_row(self, project: Project, filename: str) -> Optional[sqlite3.Row]:
        self.sync_document_state(project)
        with self._connect(project) as conn:
            return conn.execute("SELECT * FROM documents WHERE filename=?", (filename,)).fetchone()

    def document_needs_index(self, project: Project, file_path: Path) -> Tuple[bool, Dict[str, Any]]:
        self.sync_document_state(project)
        meta = self.file_meta(file_path)
        row = self.get_document_row(project, file_path.name)
        if row is None:
            return True, meta
        needs = (
            row["file_hash"] != meta["file_hash"]
            or int(row["chunk_count"] or 0) <= 0
            or str(row["status"] or "") != "indexed"
        )
        return needs, meta

    def replace_document_chunks(
        self,
        project: Project,
        file_path: Path,
        records: List[Dict[str, Any]],
        embeddings: List[List[float]],
        loader_report: str,
        meta: Dict[str, Any],
    ) -> List[Chunk]:
        self.sync_document_state(project)
        now = time.time()
        chunks: List[Chunk] = []
        with self._connect(project) as conn:
            row = conn.execute("SELECT id FROM documents WHERE filename=?", (file_path.name,)).fetchone()
            if row is None:
                doc_id = uuid.uuid4().hex
                conn.execute(
                    """
                    INSERT INTO documents(id, filename, rel_path, ext, enabled, added_at, status)
                    VALUES(?, ?, ?, ?, requirements1.txt, ?, 'pending')
                    """,
                    (doc_id, file_path.name, str(Path("docs") / file_path.name), file_path.suffix.lower(), now),
                )
            else:
                doc_id = row["id"]

            conn.execute("DELETE FROM chunks WHERE document_id=?", (doc_id,))

            for record, embedding in zip(records, embeddings):
                chunk = Chunk(
                    id=uuid.uuid4().hex,
                    source_file=record["source_file"],
                    source_path=record["source_path"],
                    page=record.get("page"),
                    text=record["text"],
                    embedding=embedding,
                    doc_type=record.get("doc_type", "text"),
                    sheet=record.get("sheet"),
                    row=record.get("row"),
                    section=record.get("section"),
                )
                chunks.append(chunk)
                conn.execute(
                    """
                    INSERT INTO chunks(
                        id, document_id, source_file, source_path, doc_type, page, sheet, row_number,
                        section, text, embedding_json, char_count, created_at
                    ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        chunk.id,
                        doc_id,
                        chunk.source_file,
                        chunk.source_path,
                        chunk.doc_type,
                        chunk.page,
                        chunk.sheet,
                        chunk.row,
                        chunk.section,
                        chunk.text,
                        json.dumps(chunk.embedding, ensure_ascii=False),
                        len(chunk.text),
                        now,
                    ),
                )

            conn.execute(
                """
                UPDATE documents
                SET file_hash=?, size_bytes=?, modified_at=?, indexed_at=?, status='indexed',
                    chunk_count=?, loader_report=?
                WHERE id=?
                """,
                (
                    meta.get("file_hash"),
                    meta.get("size_bytes"),
                    meta.get("modified_at"),
                    now,
                    len(chunks),
                    loader_report,
                    doc_id,
                ),
            )
            conn.commit()
        return chunks

    def mark_document_error(self, project: Project, filename: str, error: str) -> None:
        self.sync_document_state(project)
        with self._connect(project) as conn:
            conn.execute(
                "UPDATE documents SET status='error', loader_report=? WHERE filename=?",
                (error, filename),
            )
            conn.commit()

    def load_chunks(self, project: Project) -> List[Chunk]:
        self.sync_document_state(project)
        chunks: List[Chunk] = []
        with self._connect(project) as conn:
            rows = conn.execute(
                """
                SELECT source_file, source_path, doc_type, page, sheet, row_number, section, text, embedding_json, id
                FROM chunks
                ORDER BY source_file, page, sheet, row_number, id
                """
            ).fetchall()
        for row in rows:
            try:
                embedding = json.loads(row["embedding_json"])
                chunks.append(Chunk(
                    id=row["id"],
                    source_file=row["source_file"],
                    source_path=row["source_path"],
                    page=row["page"],
                    text=row["text"],
                    embedding=embedding,
                    doc_type=row["doc_type"] or "text",
                    sheet=row["sheet"],
                    row=row["row_number"],
                    section=row["section"],
                ))
            except Exception:
                pass
        return chunks

    def save_chunks(self, project: Project, chunks: List[Chunk]) -> None:
        """
        Полная пересборка chunks. Оставлено для совместимости, но основной путь 0.5 — replace_document_chunks().
        """
        self.sync_document_state(project)
        now = time.time()
        with self._connect(project) as conn:
            conn.execute("DELETE FROM chunks")
            for chunk in chunks:
                row = conn.execute("SELECT id FROM documents WHERE filename=?", (chunk.source_file,)).fetchone()
                if row is None:
                    doc_id = uuid.uuid4().hex
                    conn.execute(
                        """
                        INSERT INTO documents(id, filename, rel_path, ext, enabled, added_at, status)
                        VALUES(?, ?, ?, ?, requirements1.txt, ?, 'indexed')
                        """,
                        (doc_id, chunk.source_file, str(Path("docs") / chunk.source_file), Path(chunk.source_file).suffix.lower(), now),
                    )
                else:
                    doc_id = row["id"]
                conn.execute(
                    """
                    INSERT INTO chunks(id, document_id, source_file, source_path, doc_type, page, sheet, row_number, section, text, embedding_json, char_count, created_at)
                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """,
                    (
                        chunk.id, doc_id, chunk.source_file, chunk.source_path, chunk.doc_type,
                        chunk.page, chunk.sheet, chunk.row, chunk.section, chunk.text,
                        json.dumps(chunk.embedding, ensure_ascii=False), len(chunk.text), now,
                    ),
                )
            conn.execute("UPDATE documents SET chunk_count=(SELECT COUNT(*) FROM chunks WHERE chunks.document_id=documents.id)")
            conn.commit()

    def save_index_report(self, project: Project, report: List[str]) -> None:
        with self._connect(project) as conn:
            conn.execute("DELETE FROM index_report")
            for i, line in enumerate(report):
                conn.execute("INSERT INTO index_report(position, line) VALUES(?, ?)", (i, str(line)))
            conn.commit()

    def load_index_report(self, project: Project) -> List[str]:
        with self._connect(project) as conn:
            rows = conn.execute("SELECT line FROM index_report ORDER BY position").fetchall()
            return [row["line"] for row in rows]

    def load_document_summary(self, project: Project, filename: str) -> str:
        self.sync_document_state(project)
        with self._connect(project) as conn:
            row = conn.execute("SELECT summary FROM documents WHERE filename=?", (filename,)).fetchone()
            return str(row["summary"] or "") if row else ""

    def load_chunks_for_documents(self, project: Project, filenames: List[str], active_only: bool = True) -> List[Chunk]:
        wanted = set(filenames)
        if not wanted:
            return []
        chunks = self.load_chunks(project)
        if active_only:
            active = self.active_document_names(project)
            wanted = wanted & active
        return [chunk for chunk in chunks if chunk.source_file in wanted]

    def load_file_summaries(self, project: Project) -> Dict[str, Any]:
        self.sync_document_state(project)
        files: Dict[str, Any] = {}
        with self._connect(project) as conn:
            rows = conn.execute(
                "SELECT filename, loader_report, summary FROM documents WHERE summary IS NOT NULL AND summary != '' ORDER BY lower(filename)"
            ).fetchall()
            for row in rows:
                files[row["filename"]] = {
                    "file": row["filename"],
                    "loader_report": row["loader_report"] or "",
                    "summary": row["summary"] or "",
                }
        return {
            "files": files,
            "overall": self.get_setting(project, "overall_summary", ""),
        }

    def save_file_summaries(self, project: Project, data: Dict[str, Any]) -> None:
        files = data.get("files", {}) if isinstance(data, dict) else {}
        overall = data.get("overall", "") if isinstance(data, dict) else ""
        with self._connect(project) as conn:
            for filename, item in files.items():
                summary = item.get("summary", "") if isinstance(item, dict) else str(item)
                loader_report = item.get("loader_report", "") if isinstance(item, dict) else ""
                conn.execute(
                    "UPDATE documents SET summary=?, summary_updated_at=?, loader_report=COALESCE(NULLIF(?, ''), loader_report) WHERE filename=?",
                    (summary, time.time(), loader_report, filename),
                )
            conn.execute(
                "INSERT INTO settings(key, value) VALUES('overall_summary', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
                (overall,),
            )
            conn.commit()

    def update_file_summaries(self, project: Project, new_items: Dict[str, Any], overall: Optional[str] = None) -> None:
        with self._connect(project) as conn:
            for filename, item in new_items.items():
                summary = item.get("summary", "") if isinstance(item, dict) else str(item)
                loader_report = item.get("loader_report", "") if isinstance(item, dict) else ""
                conn.execute(
                    "UPDATE documents SET summary=?, summary_updated_at=?, loader_report=COALESCE(NULLIF(?, ''), loader_report) WHERE filename=?",
                    (summary, time.time(), loader_report, filename),
                )
            if overall is not None:
                conn.execute(
                    "INSERT INTO settings(key, value) VALUES('overall_summary', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
                    (overall,),
                )
            conn.commit()


# =========================
# Document loading
# =========================

class DocumentLoader:
    def __init__(self, chunk_max_chars: int = 1800, chunk_overlap: int = 250):
        self.chunk_max_chars = chunk_max_chars
        self.chunk_overlap = chunk_overlap

    def make_chunks(self, text: str) -> List[str]:
        return chunk_text(text, max_chars=self.chunk_max_chars, overlap=self.chunk_overlap)

    def load_file(self, file_path: Path) -> Tuple[List[Dict[str, Any]], str]:
        suffix = file_path.suffix.lower()
        if suffix in {".txt", ".md"}:
            return self._load_text(file_path)
        if suffix == ".pdf":
            return self._load_pdf(file_path)
        if suffix == ".docx":
            return self._load_docx(file_path)
        if suffix == ".xlsx":
            return self._load_xlsx(file_path)
        if suffix == ".csv":
            return self._load_csv(file_path)
        return [], f"{file_path.name}: неподдерживаемый формат"

    def _make_record(
        self,
        file_path: Path,
        text: str,
        doc_type: str,
        page: Optional[int] = None,
        sheet: Optional[str] = None,
        row: Optional[int] = None,
        section: Optional[str] = None,
    ) -> Dict[str, Any]:
        return {
            "source_file": file_path.name,
            "source_path": str(file_path),
            "page": page,
            "text": text,
            "doc_type": doc_type,
            "sheet": sheet,
            "row": row,
            "section": section,
        }

    def _load_text(self, file_path: Path) -> Tuple[List[Dict[str, Any]], str]:
        text = read_text_file(file_path)
        records = [self._make_record(file_path, ch, "text") for ch in self.make_chunks(text)]
        return records, f"TEXT {file_path.name}: chunks {len(records)}"

    def _load_pdf(self, file_path: Path) -> Tuple[List[Dict[str, Any]], str]:
        pages, method, total_pages = self._extract_pdf_pages(file_path)
        records = []
        for page_data in pages:
            for ch in self.make_chunks(page_data["text"]):
                records.append(self._make_record(file_path, ch, "pdf_page", page=page_data["page"]))
        empty = max(total_pages - len(pages), 0)
        if not pages:
            report = f"PDF {file_path.name}: текст не найден. Возможно, это скан или защищённый PDF"
        else:
            report = f"PDF {file_path.name}: метод {method}, страниц {total_pages}, с текстом {len(pages)}, пустых {empty}, chunks {len(records)}"
        return records, report

    def _extract_pdf_pages(self, path: Path) -> Tuple[List[Dict[str, Any]], str, int]:
        pages: List[Dict[str, Any]] = []
        total_pages = 0
        try:
            doc = fitz.open(str(path))
            total_pages = len(doc)
            for page_index in range(len(doc)):
                page = doc[page_index]
                text = (page.get_text("text") or "").strip()
                if text:
                    pages.append({"page": page_index + 1, "text": text})
            doc.close()
            if pages:
                return pages, "PyMuPDF", total_pages
        except Exception:
            pass

        try:
            reader = PdfReader(str(path))
            total_pages = max(total_pages, len(reader.pages))
            pages = []
            for i, page in enumerate(reader.pages, start=1):
                text = (page.extract_text() or "").strip()
                if text:
                    pages.append({"page": i, "text": text})
            if pages:
                return pages, "pypdf", total_pages
        except Exception:
            pass

        return [], "нет текста", total_pages

    def _load_docx(self, file_path: Path) -> Tuple[List[Dict[str, Any]], str]:
        records = []
        document = Document(str(file_path))
        paragraphs = [p.text.strip() for p in document.paragraphs if p.text.strip()]
        full_text = "\n\n".join(paragraphs)
        for ch in self.make_chunks(full_text):
            records.append(self._make_record(file_path, ch, "docx_text"))

        table_rows = 0
        for table_index, table in enumerate(document.tables, start=1):
            rows = list(table.rows)
            if not rows:
                continue
            headers = [clean_one_line(cell.text) for cell in rows[0].cells]
            if not any(headers):
                headers = [f"Колонка {i + 1}" for i in range(len(rows[0].cells))]
            summary = f"DOCX таблица {table_index}. Колонки: {', '.join(headers)}. Строк данных: {max(len(rows)-1, 0)}."
            records.append(self._make_record(file_path, summary, "table_summary", section=f"table_{table_index}"))
            for row_idx, row in enumerate(rows[1:], start=2):
                values = [clean_one_line(cell.text) for cell in row.cells]
                text = row_to_text(headers, values)
                if text:
                    table_rows += 1
                    records.append(self._make_record(file_path, text, "table_row", row=row_idx, section=f"table_{table_index}"))
        return records, f"DOCX {file_path.name}: paragraphs {len(paragraphs)}, tables {len(document.tables)}, table rows {table_rows}, chunks {len(records)}"

    def _load_xlsx(self, file_path: Path) -> Tuple[List[Dict[str, Any]], str]:
        records = []
        wb = load_workbook(filename=str(file_path), read_only=True, data_only=True)
        sheet_count = 0
        total_rows_indexed = 0
        try:
            for ws in wb.worksheets:
                sheet_count += 1
                rows_iter = ws.iter_rows(values_only=True)
                rows: List[Tuple[Any, ...]] = []
                for idx, row in enumerate(rows_iter, start=1):
                    if idx > MAX_TABLE_ROWS_PER_SHEET + 1:
                        break
                    if row and any(safe_cell(v) for v in row):
                        rows.append(row)
                if not rows:
                    records.append(self._make_record(file_path, f"Лист {ws.title}: пустой лист.", "sheet_summary", sheet=ws.title))
                    continue
                headers = [safe_cell(v) for v in rows[0]]
                if not any(headers):
                    headers = [f"Колонка {i + 1}" for i in range(len(rows[0]))]
                data_rows = rows[1:]
                non_empty_count = 0
                sample_texts = []
                for real_idx, row_values in enumerate(data_rows, start=2):
                    text = row_to_text(headers, list(row_values))
                    if not text:
                        continue
                    non_empty_count += 1
                    total_rows_indexed += 1
                    if len(sample_texts) < 3:
                        sample_texts.append(text)
                    records.append(self._make_record(file_path, text, "table_row", sheet=ws.title, row=real_idx))
                summary = (
                    f"Excel файл {file_path.name}. Лист: {ws.title}. "
                    f"Колонки: {', '.join([h for h in headers if h])}. "
                    f"Строк данных в индексе: {non_empty_count}. "
                    f"Примеры строк: {' | '.join(sample_texts) if sample_texts else 'нет'}"
                )
                records.insert(max(len(records) - non_empty_count, 0), self._make_record(file_path, summary, "sheet_summary", sheet=ws.title))
        finally:
            wb.close()
        return records, f"XLSX {file_path.name}: sheets {sheet_count}, rows indexed {total_rows_indexed}, chunks {len(records)}"

    def _load_csv(self, file_path: Path) -> Tuple[List[Dict[str, Any]], str]:
        raw = read_csv_text(file_path)
        sample = raw[:4096]
        try:
            dialect = csv.Sniffer().sniff(sample)
        except Exception:
            dialect = csv.excel
        reader = csv.reader(raw.splitlines(), dialect)
        rows = [row for row in reader if row and any(cell.strip() for cell in row)]
        records = []
        if not rows:
            return records, f"CSV {file_path.name}: пустой файл"
        headers = [cell.strip() for cell in rows[0]]
        if not any(headers):
            headers = [f"Колонка {i + 1}" for i in range(len(rows[0]))]
        sample_texts = []
        count = 0
        for row_idx, row in enumerate(rows[1:MAX_TABLE_ROWS_PER_SHEET + 1], start=2):
            text = row_to_text(headers, row)
            if not text:
                continue
            count += 1
            if len(sample_texts) < 3:
                sample_texts.append(text)
            records.append(self._make_record(file_path, text, "table_row", row=row_idx))
        summary = (
            f"CSV файл {file_path.name}. Колонки: {', '.join(headers)}. "
            f"Строк данных в индексе: {count}. Примеры: {' | '.join(sample_texts) if sample_texts else 'нет'}"
        )
        records.insert(0, self._make_record(file_path, summary, "sheet_summary"))
        return records, f"CSV {file_path.name}: rows indexed {count}, chunks {len(records)}"


# =========================
# LM Studio client
# =========================

class LMStudioClient:
    def __init__(self, base_url: str):
        self.base_url = base_url.rstrip("/")
        self.client = OpenAI(base_url=self.base_url, api_key="lm-studio")

    def list_models(self) -> List[str]:
        models = self.client.models.list()
        return [model.id for model in models.data]

    def embed_texts(self, model: str, texts: List[str], batch_size: int = 16) -> List[List[float]]:
        embeddings = []
        for i in range(0, len(texts), batch_size):
            batch = texts[i:i + batch_size]
            response = self.client.embeddings.create(model=model, input=batch)
            for item in response.data:
                embeddings.append(item.embedding)
        return embeddings

    def chat(self, model: str, messages: List[Dict[str, str]], temperature: float = 0.2) -> str:
        response = self.client.chat.completions.create(model=model, messages=messages, temperature=temperature)
        return response.choices[0].message.content or ""

    def test_chat_model(self, model: str) -> str:
        return self.chat(model=model, messages=[{"role": "user", "content": "Ответь одним коротким предложением: тест LM Studio прошёл."}], temperature=0.1)

    def test_embedding_model(self, model: str) -> Dict[str, Any]:
        texts = ["Кошка сидит на окне.", "Кот смотрит в окно.", "Космический корабль летит к Марсу."]
        vectors = self.embed_texts(model, texts)
        return {"dimension": len(vectors[0]), "similar_close": cosine_similarity(vectors[0], vectors[1]), "similar_far": cosine_similarity(vectors[0], vectors[2])}


# =========================
# Indexing / Retrieval
# =========================

class IndexService:
    def __init__(
        self,
        store: ProjectStore,
        lm: LMStudioClient,
        embedding_model: str,
        chunk_max_chars: int = 1800,
        chunk_overlap: int = 250,
    ):
        self.store = store
        self.lm = lm
        self.embedding_model = embedding_model
        self.chunk_max_chars = chunk_max_chars
        self.chunk_overlap = chunk_overlap
        self.loader = DocumentLoader(chunk_max_chars=chunk_max_chars, chunk_overlap=chunk_overlap)

    def build_index(self, project: Project) -> Tuple[List[Chunk], List[str]]:
        files = self.store.list_document_files(project, active_only=True)
        report: List[str] = ["SQLite storage: ON", "Инкрементальная индексация: ON"]

        signature = self.store.get_index_signature(
            self.embedding_model,
            self.chunk_max_chars,
            self.chunk_overlap,
        )
        settings_changed = self.store.ensure_index_signature(project, signature)
        if settings_changed:
            report.append("Настройки индекса изменились: старые chunks сброшены, активные документы будут переиндексированы.")

        indexed_now = 0
        skipped = 0
        errors = 0

        for file_path in files:
            try:
                needs_index, meta = self.store.document_needs_index(project, file_path)
                if not needs_index:
                    skipped += 1
                    row = self.store.get_document_row(project, file_path.name)
                    chunk_count = int(row["chunk_count"] or 0) if row else 0
                    report.append(f"SKIP {file_path.name}: файл не изменился, chunks {chunk_count}")
                    continue

                records, line = self.loader.load_file(file_path)
                if not records:
                    self.store.replace_document_chunks(project, file_path, [], [], line, meta)
                    report.append(line)
                    indexed_now += 1
                    continue

                texts = [r["text"] for r in records]
                embeddings = self.lm.embed_texts(self.embedding_model, texts)
                chunks = self.store.replace_document_chunks(project, file_path, records, embeddings, line, meta)
                indexed_now += 1
                report.append(line)
                report.append(f"INDEXED {file_path.name}: chunks {len(chunks)}")
            except Exception as e:
                errors += 1
                self.store.mark_document_error(project, file_path.name, str(e))
                report.append(f"ERROR {file_path.name}: {e}")

        all_chunks = self.store.load_chunks(project)
        active_names = self.store.active_document_names(project)
        active_chunks = [chunk for chunk in all_chunks if chunk.source_file in active_names]
        stats = chunk_statistics(active_chunks)
        report.append(
            "Chunk stats active: "
            f"count {stats['count']}, "
            f"avg {stats['avg_len']:.0f} chars, "
            f"min {stats['min_len']}, "
            f"max {stats['max_len']}, "
            f"short<120 {stats['short_count']}, "
            f"long>3500 {stats['long_count']}, "
            f"settings max_chars={self.chunk_max_chars}, overlap={self.chunk_overlap}"
        )
        report.append(f"Итог: проиндексировано сейчас {indexed_now}, пропущено без изменений {skipped}, ошибок {errors}")
        self.store.save_index_report(project, report)
        return active_chunks, report


class RetrievalService:
    def __init__(self, store: ProjectStore, lm: LMStudioClient, embedding_model: str):
        self.store = store
        self.lm = lm
        self.embedding_model = embedding_model

    def search(self, project: Project, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
        chunks = self.store.load_chunks(project)
        active_docs = self.store.active_document_names(project)
        chunks = [chunk for chunk in chunks if chunk.source_file in active_docs]
        if not chunks:
            return []
        query_embedding = self.lm.embed_texts(self.embedding_model, [query])[0]
        scored = []
        for chunk in chunks:
            score = cosine_similarity(query_embedding, chunk.embedding)
            scored.append({"score": score, "chunk": chunk})
        scored.sort(key=lambda x: x["score"], reverse=True)
        return scored[:top_k]


# =========================
# Agents
# =========================

class AgentPrompts:
    QUERY_REWRITER_SYSTEM = """
Ты агент переписывания поисковых запросов.
Твоя задача — превратить вопрос пользователя в короткий и точный поисковый запрос для поиска по локальным документам.
Не отвечай на вопрос. Не добавляй пояснений. Верни только улучшенный поисковый запрос.
""".strip()

    RESEARCHER_SYSTEM = """
Ты исследовательский агент. Работай только с предоставленными фрагментами документов.
Извлеки факты, которые помогут ответить на вопрос пользователя. Не выдумывай.
Если данных мало, так и напиши. Верни структурированные заметки.
""".strip()

    ANSWER_SYSTEM = """
Ты локальный ассистент для работы с документами.
Отвечай только на основе предоставленных источников. Если данных недостаточно, прямо скажи, что в источниках этого нет.
Не выдумывай факты. Пиши понятно и полезно. В конце добавь раздел "Источники".
""".strip()

    CRITIC_SYSTEM = """
Ты агент-критик и проверяющий. Проверь ответ по предоставленным источникам.
Удали или исправь всё, что не подтверждается источниками. Не добавляй новые неподтверждённые факты.
Если данных в источниках нет, ответ должен честно это сказать. Верни только финальную исправленную версию ответа.
""".strip()

    OVERVIEW_SYSTEM = """
Ты ассистент для обзора локальных документов.
На основе предоставленных фрагментов сделай обзор проекта: перечисли документы, тему каждого документа, важные таблицы/листы, ключевые выводы и что можно спросить дальше.
Не выдумывай того, чего нет в источниках. Если документ представлен только частично, скажи, что обзор предварительный.
""".strip()


class AgentChainService:
    def __init__(self, store: ProjectStore, lm: LMStudioClient, chat_model: str, embedding_model: str, rewriter_model: str, critic_model: str, min_score: float):
        self.store = store
        self.lm = lm
        self.chat_model = chat_model
        self.embedding_model = embedding_model
        self.rewriter_model = rewriter_model or chat_model
        self.critic_model = critic_model or chat_model
        self.min_score = min_score

    def _chunk_location(self, chunk: Chunk) -> str:
        parts = [chunk.source_file]
        if chunk.page:
            parts.append(f"стр. {chunk.page}")
        if chunk.sheet:
            parts.append(f"лист {chunk.sheet}")
        if chunk.row:
            parts.append(f"строка {chunk.row}")
        if chunk.section:
            parts.append(chunk.section)
        return ", ".join(parts)

    def _format_context(self, results: List[Dict[str, Any]]) -> str:
        blocks = []
        for i, item in enumerate(results, start=1):
            chunk: Chunk = item["chunk"]
            blocks.append(f"[Источник {i}: {self._chunk_location(chunk)} | тип {chunk.doc_type}]\n{chunk.text}")
        return "\n\n---\n\n".join(blocks)

    def _format_source_list(self, results: List[Dict[str, Any]]) -> str:
        if not results:
            return "Нет найденных фрагментов."
        lines = []
        for i, item in enumerate(results, start=1):
            chunk: Chunk = item["chunk"]
            lines.append(f"{i}. {self._chunk_location(chunk)} — score {item['score']:.3f} | {chunk.doc_type}")
        return "\n".join(lines)

    def _debug_sources(self, results: List[Dict[str, Any]], title: str = "Найденные фрагменты") -> str:
        if not results:
            return f"{title}: нет."
        blocks = [title]
        for i, item in enumerate(results, start=1):
            chunk: Chunk = item["chunk"]
            blocks.append(f"\n[{i}] score {item['score']:.3f} | {self._chunk_location(chunk)} | {chunk.doc_type}\n{make_snippet(chunk.text)}")
        return "\n".join(blocks)

    def _diagnostic_footer(self, mode_name: str, raw_results: List[Dict[str, Any]], accepted_results: List[Dict[str, Any]], extra: Optional[str] = None) -> str:
        best_score = raw_results[0]["score"] if raw_results else None
        footer = [
            "---",
            "RAG-диагностика:",
            f"Режим: {mode_name}",
            f"Лучший score: {best_score:.3f}" if best_score is not None else "Лучший score: нет",
            f"Уверенность: {confidence_label(best_score)}",
            f"Минимальный порог: {self.min_score:.2f}",
            f"Принято фрагментов: {len(accepted_results)} из {len(raw_results)}",
        ]
        if extra:
            footer.append(extra.strip())
        footer.append("\nНайденные фрагменты:")
        footer.append(self._format_source_list(raw_results))
        return "\n".join(footer)

    def _retrieve_filtered(self, project: Project, query: str, top_k: int) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], str]:
        retriever = RetrievalService(self.store, self.lm, self.embedding_model)
        raw = retriever.search(project, query, top_k=top_k)
        accepted = [item for item in raw if item["score"] >= self.min_score]
        context = self._format_context(accepted)
        return raw, accepted, context

    def rewrite_query(self, question: str) -> str:
        rewritten = self.lm.chat(self.rewriter_model, [
            {"role": "system", "content": AgentPrompts.QUERY_REWRITER_SYSTEM},
            {"role": "user", "content": question},
        ], temperature=0.1).strip()
        return rewritten.replace('"', '').strip() or question

    def researcher_notes(self, question: str, context: str) -> str:
        prompt = f"Вопрос пользователя:\n{question}\n\nФрагменты документов:\n{context}\n\nИзвлеки только релевантные факты и наблюдения."
        return self.lm.chat(self.chat_model, [
            {"role": "system", "content": AgentPrompts.RESEARCHER_SYSTEM},
            {"role": "user", "content": prompt},
        ], temperature=0.2).strip()

    def draft_answer(self, question: str, context: str, notes: Optional[str] = None) -> str:
        user_prompt = f"Вопрос пользователя:\n{question}\n\n"
        if notes:
            user_prompt += f"Заметки исследовательского агента:\n{notes}\n\n"
        user_prompt += f"Фрагменты документов:\n{context}\n\nСформируй точный финальный ответ на русском языке."
        return self.lm.chat(self.chat_model, [
            {"role": "system", "content": AgentPrompts.ANSWER_SYSTEM},
            {"role": "user", "content": user_prompt},
        ], temperature=0.2).strip()

    def critic_review(self, question: str, context: str, draft: str) -> str:
        prompt = f"Вопрос пользователя:\n{question}\n\nИсточники:\n{context}\n\nЧерновик ответа:\n{draft}\n\nПроверь черновик по источникам и верни финальную исправленную версию."
        return self.lm.chat(self.critic_model, [
            {"role": "system", "content": AgentPrompts.CRITIC_SYSTEM},
            {"role": "user", "content": prompt},
        ], temperature=0.0).strip()

    def _no_context_answer(self, mode_name: str, raw: List[Dict[str, Any]], accepted: List[Dict[str, Any]]) -> Dict[str, str]:
        answer = "Я не нашёл достаточно релевантных фрагментов в индексе проекта.\n\nМожно снизить порог релевантности, увеличить число фрагментов или переиндексировать документы."
        return {"answer": answer + "\n\n" + self._diagnostic_footer(mode_name, raw, accepted), "debug": self._debug_sources(raw, "Сырые результаты поиска")}

    def answer_basic(self, project: Project, question: str, top_k: int) -> Dict[str, str]:
        raw, accepted, context = self._retrieve_filtered(project, question, top_k)
        if not raw:
            return {"answer": "В проекте пока нет индекса. Добавь документы и нажми «Переиндексировать».", "debug": "Фрагменты не найдены: индекс пуст."}
        if not accepted:
            return self._no_context_answer("Обычный RAG", raw, accepted)
        draft = self.draft_answer(question, context)
        return {"answer": draft + "\n\n" + self._diagnostic_footer("Обычный RAG", raw, accepted), "debug": self._debug_sources(raw)}

    def answer_with_critic(self, project: Project, question: str, top_k: int) -> Dict[str, str]:
        raw, accepted, context = self._retrieve_filtered(project, question, top_k)
        if not raw:
            return {"answer": "В проекте пока нет индекса. Добавь документы и нажми «Переиндексировать».", "debug": "Фрагменты не найдены: индекс пуст."}
        if not accepted:
            return self._no_context_answer("RAG + критик", raw, accepted)
        draft = self.draft_answer(question, context)
        final_answer = self.critic_review(question, context, draft)
        return {"answer": final_answer + "\n\n" + self._diagnostic_footer("RAG + критик", raw, accepted), "debug": self._debug_sources(raw)}

    def answer_chain(self, project: Project, question: str, top_k: int) -> Dict[str, str]:
        rewritten_query = self.rewrite_query(question)
        raw, accepted, context = self._retrieve_filtered(project, rewritten_query, top_k)
        if not raw:
            return {"answer": f"В проекте пока нет индекса или агент не нашёл фрагменты.\n\nЗапрос агента: {rewritten_query}", "debug": "Фрагменты не найдены: индекс пуст."}
        if not accepted:
            result = self._no_context_answer("Цепочка агентов", raw, accepted)
            result["answer"] += f"\n\nЗапрос агента: {rewritten_query}"
            return result
        notes = self.researcher_notes(question, context)
        draft = self.draft_answer(question, context, notes=notes)
        final_answer = self.critic_review(question, context, draft)
        extra = f"Запрос агента: {rewritten_query}"
        debug = f"Агентная цепочка\n\nrequirements1.txt. Query Rewriter:\n{rewritten_query}\n\n2. Researcher Notes:\n{notes}\n\n{self._debug_sources(raw)}"
        return {"answer": final_answer + "\n\n" + self._diagnostic_footer("Цепочка агентов", raw, accepted, extra=extra), "debug": debug}

    def _chunk_location(self, chunk: Chunk) -> str:
        parts = [chunk.source_file]
        if chunk.page:
            parts.append(f"стр. {chunk.page}")
        if chunk.sheet:
            parts.append(f"лист {chunk.sheet}")
        if chunk.row:
            parts.append(f"строка {chunk.row}")
        return ", ".join(parts)

    def _select_review_chunks(self, chunks: List[Chunk], per_file_limit: int = 8) -> List[Chunk]:
        by_doc: Dict[str, List[Chunk]] = {}
        for chunk in chunks:
            by_doc.setdefault(chunk.source_file, []).append(chunk)

        selected: List[Chunk] = []
        priority_re = re.compile(
            r"abstract|summary|conclusion|discussion|introduction|result|итог|вывод|заключ|резюме|цель|задач",
            re.IGNORECASE,
        )

        for filename, doc_chunks in by_doc.items():
            priorities = [c for c in doc_chunks if c.doc_type in {"sheet_summary", "table_summary"}]
            priorities.extend([c for c in doc_chunks if priority_re.search(c.text)])
            priorities.extend(doc_chunks[:4])

            seen = set()
            count = 0
            for chunk in priorities:
                if chunk.id in seen:
                    continue
                seen.add(chunk.id)
                selected.append(chunk)
                count += 1
                if count >= per_file_limit:
                    break

        return selected[:MAX_OVERVIEW_CHUNKS]

    def _build_review_context(self, project: Project, filenames: Optional[List[str]] = None, per_file_limit: int = 8) -> Tuple[str, str, List[str]]:
        active_docs = self.store.active_document_names(project)
        summaries_data = self.store.load_file_summaries(project)
        summaries = summaries_data.get("files", {}) if isinstance(summaries_data, dict) else {}

        if filenames:
            target_docs = [name for name in filenames if name in active_docs]
        else:
            target_docs = sorted(active_docs)

        chunks = self.store.load_chunks_for_documents(project, target_docs, active_only=True)
        selected = self._select_review_chunks(chunks, per_file_limit=per_file_limit)

        blocks: List[str] = []
        used_locations: List[str] = []

        for filename in target_docs:
            item = summaries.get(filename)
            if item and item.get("summary"):
                blocks.append(f"[Резюме файла: {filename}]\n{item.get('summary')}")

        for i, chunk in enumerate(selected, start=1):
            location = self._chunk_location(chunk)
            used_locations.append(location)
            blocks.append(f"[Фрагмент {i}: {location}; тип {chunk.doc_type}]\n{make_snippet(chunk.text, 1400)}")

        return "\n\n---\n\n".join(blocks), "\n".join(f"- {x}" for x in used_locations), target_docs

    def answer_compare(self, project: Project, question: str, filenames: List[str]) -> Dict[str, str]:
        if not filenames or len(filenames) < 2:
            return {
                "answer": "Для сравнения нужно выделить минимум два активных документа в списке слева.",
                "debug": "Выдели документы мышкой. Галочки отвечают за активность, выделение — за сравнение.",
            }

        context, used, target_docs = self._build_review_context(
            project,
            filenames=filenames,
            per_file_limit=MAX_COMPARE_CHUNKS_PER_FILE,
        )

        if not context.strip():
            return {
                "answer": "Не нашёл резюме или индексированных фрагментов для выбранных активных документов. Сначала сделай резюме проекта и переиндексацию.",
                "debug": "Выбранные документы:\n" + "\n".join(f"- {name}" for name in filenames),
            }

        user_task = question.strip() or "Сравни выбранные документы: тема, назначение, ключевые факты, различия, совпадения и практический вывод."

        prompt = f"""
Задача пользователя:
{user_task}

Выбранные документы:
{chr(10).join('- ' + name for name in target_docs)}

Материалы для сравнения:
{context}

Сделай сравнение на русском языке.
Структура ответа:
requirements1.txt. Кратко: что это за документы.
2. Сравнительная таблица: документ / тема / важные данные / вывод.
3. Что общего.
4. Главные различия.
5. Что важно проверить вручную.
6. Итоговый вывод.
Не выдумывай факты, которых нет в материалах.
""".strip()

        answer = self.lm.chat(
            self.chat_model,
            [
                {"role": "system", "content": "Ты аналитик документов. Сравнивай только по предоставленным материалам. Если данных мало, прямо скажи об этом."},
                {"role": "user", "content": prompt},
            ],
            temperature=0.2,
        ).strip()

        debug = "Сравнение использовало документы:\n" + "\n".join(f"- {name}" for name in target_docs)
        if used:
            debug += "\n\nИспользованные обзорные фрагменты:\n" + used

        return {
            "answer": answer + "\n\n---\nРежим: Сравнение документов\nДокументов: " + str(len(target_docs)),
            "debug": debug,
        }

    def answer_overview(self, project: Project) -> Dict[str, str]:
        chunks = self.store.load_chunks(project)
        active_docs = self.store.active_document_names(project)
        chunks = [chunk for chunk in chunks if chunk.source_file in active_docs]
        if not chunks:
            return {"answer": "В проекте пока нет активного индекса. Включи документы галочками и нажми «Переиндексировать».", "debug": "Активные фрагменты не найдены."}
        by_doc: Dict[str, List[Chunk]] = {}
        for chunk in chunks:
            by_doc.setdefault(chunk.source_file, []).append(chunk)

        selected: List[Chunk] = []
        for file_name, doc_chunks in by_doc.items():
            summaries = [c for c in doc_chunks if c.doc_type in {"sheet_summary", "table_summary"}]
            selected.extend(summaries[:5])
            selected.extend(doc_chunks[:4])
            conclusion_like = [c for c in doc_chunks if re.search(r"conclusion|summary|abstract|итог|вывод|заключ", c.text, re.IGNORECASE)]
            selected.extend(conclusion_like[:3])
        # remove duplicates while preserving order
        seen = set()
        unique = []
        for c in selected:
            if c.id not in seen:
                seen.add(c.id)
                unique.append(c)
        selected = unique[:MAX_OVERVIEW_CHUNKS]

        context = "\n\n---\n\n".join([f"[Документ: {self._chunk_location(c)} | тип {c.doc_type}]\n{c.text}" for c in selected])
        prompt = f"В проекте {len(by_doc)} документов. Сделай обзор по ним.\n\nФрагменты:\n{context}"
        answer = self.lm.chat(self.chat_model, [
            {"role": "system", "content": AgentPrompts.OVERVIEW_SYSTEM},
            {"role": "user", "content": prompt},
        ], temperature=0.2).strip()
        report = self.store.load_index_report(project)
        debug = "Обзор использовал фрагменты:\n\n" + "\n".join([f"- {self._chunk_location(c)} | {c.doc_type}" for c in selected])
        if report:
            debug += "\n\nОтчёт индексации:\n" + "\n".join(report)
        return {"answer": answer + "\n\n---\nРежим: Обзор проекта\nДокументов: " + str(len(by_doc)) + f"\nИспользовано обзорных фрагментов: {len(selected)}", "debug": debug}

    def answer(self, project: Project, question: str, mode: str, top_k: int, selected_documents: Optional[List[str]] = None) -> Dict[str, str]:
        if mode == "overview":
            return self.answer_overview(project)
        if mode == "compare":
            return self.answer_compare(project, question, selected_documents or [])
        if mode == "critic":
            return self.answer_with_critic(project, question, top_k)
        if mode == "chain":
            return self.answer_chain(project, question, top_k)
        return self.answer_basic(project, question, top_k)


# =========================
# Auto summaries
# =========================

class AutoSummaryService:
    def __init__(self, lm: LMStudioClient, chat_model: str):
        self.lm = lm
        self.chat_model = chat_model
        self.loader = DocumentLoader()

    def _select_records_for_summary(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        if not records:
            return []
        selected: List[Dict[str, Any]] = []
        selected.extend([r for r in records if r.get("doc_type") in {"sheet_summary", "table_summary"}][:8])
        selected.extend(records[:6])
        keyword_re = re.compile(r"abstract|summary|conclusion|вывод|итог|заключ|назначение|основание|цель|резюме", re.IGNORECASE)
        selected.extend([r for r in records if keyword_re.search(r.get("text", ""))][:6])
        seen = set()
        unique = []
        for item in selected:
            key = (
                item.get("source_file"), item.get("page"), item.get("sheet"),
                item.get("row"), item.get("section"), item.get("text", "")[:120]
            )
            if key in seen:
                continue
            seen.add(key)
            unique.append(item)
        return unique[:MAX_AUTO_SUMMARY_RECORDS_PER_FILE]

    def _record_location(self, r: Dict[str, Any]) -> str:
        parts = [r.get("source_file", "файл")]
        if r.get("page"):
            parts.append(f"стр. {r['page']}")
        if r.get("sheet"):
            parts.append(f"лист {r['sheet']}")
        if r.get("row"):
            parts.append(f"строка {r['row']}")
        if r.get("section"):
            parts.append(str(r["section"]))
        parts.append(str(r.get("doc_type", "text")))
        return ", ".join(parts)

    def _context_from_records(self, records: List[Dict[str, Any]]) -> str:
        blocks = []
        total = 0
        for i, r in enumerate(records, start=1):
            block = f"[Фрагмент {i}: {self._record_location(r)}]\n{r.get('text', '')}"
            if total + len(block) > MAX_AUTO_SUMMARY_CONTEXT_CHARS:
                break
            blocks.append(block)
            total += len(block)
        return "\n\n---\n\n".join(blocks)

    def summarize_file(self, file_path: Path) -> Tuple[Dict[str, Any], str]:
        records, loader_report = self.loader.load_file(file_path)
        selected = self._select_records_for_summary(records)
        if not records:
            summary = (
                f"Файл {file_path.name} добавлен, но текстовые данные не извлечены. "
                "Возможные причины: сканированный PDF, защищённый файл или неподдерживаемая структура."
            )
            return {
                "file": file_path.name,
                "summary": summary,
                "loader_report": loader_report,
                "records_total": 0,
                "records_used": 0,
            }, summary
        context = self._context_from_records(selected)
        prompt = f"""
Пользователь только что загрузил файл: {file_path.name}
Отчёт чтения: {loader_report}

Фрагменты файла:
{context}

Сделай короткое первичное резюме файла на русском языке.
Формат:
requirements1.txt. Что это за файл.
2. О чём он, если можно понять.
3. Какие важные данные/таблицы/разделы видны.
4. Что полезно спросить у ассистента дальше.

Не выдумывай. Если обзор предварительный, так и скажи.
""".strip()
        summary = self.lm.chat(
            self.chat_model,
            [
                {"role": "system", "content": "Ты делаешь краткий обзор только что загруженного локального документа. Не выдумывай факты."},
                {"role": "user", "content": prompt},
            ],
            temperature=0.2,
        ).strip()
        return {
            "file": file_path.name,
            "summary": summary,
            "loader_report": loader_report,
            "records_total": len(records),
            "records_used": len(selected),
        }, summary

    def summarize_many(self, file_summaries: List[Dict[str, Any]]) -> str:
        if not file_summaries:
            return "Нет файлов для общего резюме."
        if len(file_summaries) == 1:
            return file_summaries[0].get("summary", "")
        joined = "\n\n---\n\n".join(
            f"Файл: {item.get('file')}\nОтчёт: {item.get('loader_report')}\nРезюме:\n{item.get('summary')}"
            for item in file_summaries
        )
        prompt = f"""
Пользователь загрузил несколько файлов. Ниже краткие резюме по каждому файлу:

{joined}

Сделай общее короткое резюме всей загрузки:
requirements1.txt. Сколько файлов и какие типы материалов.
2. Общая тема/направление, если видно.
3. Что в них самое полезное.
4. Какие вопросы стоит задать дальше.

Не выдумывай и не делай вид, что видел больше данных, чем указано.
""".strip()
        return self.lm.chat(
            self.chat_model,
            [
                {"role": "system", "content": "Ты объединяешь краткие резюме загруженных документов в общий обзор."},
                {"role": "user", "content": prompt},
            ],
            temperature=0.2,
        ).strip()


# =========================
# Background workers
# =========================

class DiagnosticWorker(QThread):
    finished_ok = Signal(object)
    failed = Signal(str)

    def __init__(self, base_url: str, chat_model: str, embedding_model: str):
        super().__init__()
        self.base_url = base_url
        self.chat_model = chat_model
        self.embedding_model = embedding_model

    def run(self):
        try:
            lm = LMStudioClient(self.base_url)
            models = lm.list_models()
            lines = ["LM Studio: подключение успешно.", f"Найдено моделей: {len(models)}"]
            if self.chat_model:
                try:
                    answer = lm.test_chat_model(self.chat_model)
                    lines.append(f"Chat model OK: {self.chat_model}")
                    lines.append(f"Ответ: {clean_one_line(answer)}")
                except Exception as e:
                    lines.append(f"Chat model ERROR: {self.chat_model}")
                    lines.append(str(e))
            if self.embedding_model:
                try:
                    data = lm.test_embedding_model(self.embedding_model)
                    lines.append(f"Embedding model OK: {self.embedding_model}")
                    lines.append(f"Размер вектора: {data['dimension']}")
                    lines.append(f"Кошка/кот: {data['similar_close']:.3f}")
                    lines.append(f"Кошка/космос: {data['similar_far']:.3f}")
                except Exception as e:
                    lines.append(f"Embedding model ERROR: {self.embedding_model}")
                    lines.append(str(e))
            self.finished_ok.emit({"models": models, "report": "\n".join(lines)})
        except Exception as e:
            self.failed.emit(str(e))


class IndexWorker(QThread):
    finished_ok = Signal(object)
    failed = Signal(str)

    def __init__(
        self,
        project: Project,
        base_url: str,
        embedding_model: str,
        chunk_max_chars: int,
        chunk_overlap: int,
    ):
        super().__init__()
        self.project = project
        self.base_url = base_url
        self.embedding_model = embedding_model
        self.chunk_max_chars = chunk_max_chars
        self.chunk_overlap = chunk_overlap

    def run(self):
        try:
            store = ProjectStore()
            lm = LMStudioClient(self.base_url)
            indexer = IndexService(
                store,
                lm,
                self.embedding_model,
                chunk_max_chars=self.chunk_max_chars,
                chunk_overlap=self.chunk_overlap,
            )
            chunks, report = indexer.build_index(self.project)
            self.finished_ok.emit({"message": f"Индексация завершена. Фрагментов: {len(chunks)}", "report": report})
        except Exception as e:
            self.failed.emit(str(e))


class ChatWorker(QThread):
    finished_ok = Signal(object)
    failed = Signal(str)

    def __init__(self, project: Project, question: str, base_url: str, chat_model: str, embedding_model: str, rewriter_model: str, critic_model: str, mode: str, top_k: int, min_score: float, selected_documents: Optional[List[str]] = None):
        super().__init__()
        self.project = project
        self.question = question
        self.base_url = base_url
        self.chat_model = chat_model
        self.embedding_model = embedding_model
        self.rewriter_model = rewriter_model
        self.critic_model = critic_model
        self.mode = mode
        self.top_k = top_k
        self.min_score = min_score
        self.selected_documents = selected_documents or []

    def run(self):
        try:
            store = ProjectStore()
            lm = LMStudioClient(self.base_url)
            service = AgentChainService(store, lm, self.chat_model, self.embedding_model, self.rewriter_model, self.critic_model, self.min_score)
            result = service.answer(self.project, self.question, self.mode, self.top_k, selected_documents=self.selected_documents)
            self.finished_ok.emit(result)
        except Exception as e:
            self.failed.emit(str(e))




class AutoSummaryWorker(QThread):
    finished_ok = Signal(object)
    failed = Signal(str)

    def __init__(self, project: Project, file_paths: List[str], base_url: str, chat_model: str):
        super().__init__()
        self.project = project
        self.file_paths = file_paths
        self.base_url = base_url
        self.chat_model = chat_model

    def run(self):
        try:
            store = ProjectStore()
            lm = LMStudioClient(self.base_url)
            service = AutoSummaryService(lm, self.chat_model)
            items: List[Dict[str, Any]] = []
            file_map: Dict[str, Any] = {}
            report_lines: List[str] = []
            for path_text in self.file_paths:
                file_path = Path(path_text)
                item, _summary = service.summarize_file(file_path)
                items.append(item)
                file_map[file_path.name] = item
                report_lines.append(
                    f"{file_path.name}: резюме готово, использовано фрагментов {item.get('records_used', 0)} из {item.get('records_total', 0)}"
                )
            overall = service.summarize_many(items)
            store.update_file_summaries(self.project, file_map, overall=overall)
            text_blocks = ["Авто-резюме загруженных файлов", ""]
            for item in items:
                text_blocks.append(f"## {item.get('file')}\n{item.get('summary', '')}\n")
            if len(items) > 1:
                text_blocks.append("## Общее резюме загрузки\n" + overall)
            self.finished_ok.emit({
                "message": "Авто-резюме готово.",
                "summary_text": "\n".join(text_blocks),
                "report": report_lines,
                "overall": overall,
            })
        except Exception as e:
            self.failed.emit(str(e))

# =========================
# UI
# =========================

class MainWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("Local Notebook LM Studio — v0.6 Review & Compare")
        self.resize(1520, 900)

        self.store = ProjectStore()
        self.current_project: Optional[Project] = None
        self.projects: List[Project] = []
        self.current_mode = "critic"

        self.index_worker: Optional[IndexWorker] = None
        self.chat_worker: Optional[ChatWorker] = None
        self.diagnostic_worker: Optional[DiagnosticWorker] = None
        self.summary_worker: Optional[AutoSummaryWorker] = None

        self.project_list = QListWidget()
        self.project_list.currentRowChanged.connect(self.on_project_selected)

        self.new_project_btn = QPushButton("Новый проект")
        self.new_project_btn.clicked.connect(self.create_project)
        self.add_files_btn = QPushButton("Добавить документы")
        self.add_files_btn.clicked.connect(self.add_documents)
        self.index_btn = QPushButton("Переиндексировать")
        self.index_btn.clicked.connect(self.reindex_project)

        self.project_summary_btn = QPushButton("Резюме проекта")
        self.project_summary_btn.clicked.connect(self.summarize_project_files)

        self.delete_project_btn = QPushButton("Удалить проект")
        self.delete_project_btn.clicked.connect(self.delete_current_project)

        self.delete_docs_btn = QPushButton("Удалить выбранные документы")
        self.delete_docs_btn.clicked.connect(self.delete_selected_documents)

        self._updating_docs_list = False
        self.docs_list = QListWidget()
        self.docs_list.itemChanged.connect(self.on_document_check_changed)

        self.docs_info_view = QTextEdit()
        self.docs_info_view.setReadOnly(True)
        self.docs_info_view.setMaximumHeight(130)

        left_layout = QVBoxLayout()
        left_layout.addWidget(QLabel("Проекты"))
        left_layout.addWidget(self.project_list)
        left_layout.addWidget(self.new_project_btn)
        left_layout.addWidget(self.delete_project_btn)
        left_layout.addWidget(self.add_files_btn)
        left_layout.addWidget(self.index_btn)
        left_layout.addWidget(self.project_summary_btn)
        left_layout.addWidget(QLabel("Документы проекта"))
        left_layout.addWidget(self.docs_list)
        left_layout.addWidget(self.delete_docs_btn)
        left_layout.addWidget(QLabel("Сводка по документам"))
        left_layout.addWidget(self.docs_info_view)
        left_panel = QWidget()
        left_panel.setLayout(left_layout)

        self.basic_mode_btn = self.make_mode_button("Обычный RAG", "basic")
        self.critic_mode_btn = self.make_mode_button("RAG + критик", "critic")
        self.chain_mode_btn = self.make_mode_button("Цепочка агентов", "chain")
        self.overview_mode_btn = self.make_mode_button("Обзор проекта", "overview")
        self.compare_mode_btn = self.make_mode_button("Сравнение документов", "compare")
        self.critic_mode_btn.setChecked(True)

        mode_buttons_layout = QHBoxLayout()
        mode_buttons_layout.addWidget(self.basic_mode_btn)
        mode_buttons_layout.addWidget(self.critic_mode_btn)
        mode_buttons_layout.addWidget(self.chain_mode_btn)
        mode_buttons_layout.addWidget(self.overview_mode_btn)
        mode_buttons_layout.addWidget(self.compare_mode_btn)

        self.chat_view = QTextEdit()
        self.chat_view.setReadOnly(True)
        self.question_input = QLineEdit()
        self.question_input.setPlaceholderText("Задай вопрос по документам...")
        self.question_input.returnPressed.connect(self.ask_question)
        self.ask_btn = QPushButton("Спросить")
        self.ask_btn.clicked.connect(self.ask_question)

        input_layout = QHBoxLayout()
        input_layout.addWidget(self.question_input)
        input_layout.addWidget(self.ask_btn)

        center_layout = QVBoxLayout()
        center_layout.addWidget(QLabel("Режим ответа"))
        center_layout.addLayout(mode_buttons_layout)
        center_layout.addWidget(QLabel("Чат"))
        center_layout.addWidget(self.chat_view)
        center_layout.addLayout(input_layout)
        center_panel = QWidget()
        center_panel.setLayout(center_layout)

        self.base_url_input = QLineEdit("http://localhost:1234/v1")
        self.chat_model_combo = self.make_model_combo("local-model")
        self.rewriter_model_combo = self.make_model_combo("local-model")
        self.critic_model_combo = self.make_model_combo("local-model")
        self.embedding_model_combo = self.make_model_combo("text-embedding-nomic-embed-text-v1.5")

        self.top_k_input = QSpinBox()
        self.top_k_input.setMinimum(1)
        self.top_k_input.setMaximum(50)
        self.top_k_input.setValue(7)
        self.min_score_input = QDoubleSpinBox()
        self.min_score_input.setMinimum(0.0)
        self.min_score_input.setMaximum(1.0)
        self.min_score_input.setDecimals(2)
        self.min_score_input.setSingleStep(0.05)
        self.min_score_input.setValue(0.40)

        self.chunk_size_input = QSpinBox()
        self.chunk_size_input.setMinimum(400)
        self.chunk_size_input.setMaximum(6000)
        self.chunk_size_input.setSingleStep(100)
        self.chunk_size_input.setValue(1800)

        self.chunk_overlap_input = QSpinBox()
        self.chunk_overlap_input.setMinimum(0)
        self.chunk_overlap_input.setMaximum(2000)
        self.chunk_overlap_input.setSingleStep(50)
        self.chunk_overlap_input.setValue(250)

        self.check_btn = QPushButton("Проверить LM Studio и модели")
        self.check_btn.clicked.connect(self.check_lmstudio)

        settings_form = QFormLayout()
        settings_form.addRow("LM Studio API:", self.base_url_input)
        settings_form.addRow("Answer model:", self.chat_model_combo)
        settings_form.addRow("Query Rewriter model:", self.rewriter_model_combo)
        settings_form.addRow("Critic model:", self.critic_model_combo)
        settings_form.addRow("Embedding model:", self.embedding_model_combo)
        settings_form.addRow("Фрагментов:", self.top_k_input)
        settings_form.addRow("Мин. score:", self.min_score_input)
        settings_form.addRow("Chunk size:", self.chunk_size_input)
        settings_form.addRow("Chunk overlap:", self.chunk_overlap_input)
        settings_box = QGroupBox("Настройки")
        settings_box.setLayout(settings_form)

        self.sources_view = QTextEdit()
        self.sources_view.setReadOnly(True)
        self.sources_view.setPlaceholderText("Здесь будут найденные фрагменты, отчёт Excel/PDF/DOCX/CSV и лог агентов.")
        self.log_view = QTextEdit()
        self.log_view.setReadOnly(True)
        self.log_view.setMaximumHeight(190)

        right_layout = QVBoxLayout()
        right_layout.addWidget(settings_box)
        right_layout.addWidget(self.check_btn)
        right_layout.addWidget(QLabel("Найденные фрагменты / отчёт"))
        right_layout.addWidget(self.sources_view)
        right_layout.addWidget(QLabel("Лог"))
        right_layout.addWidget(self.log_view)
        right_panel = QWidget()
        right_panel.setLayout(right_layout)

        splitter = QSplitter(Qt.Horizontal)
        splitter.addWidget(left_panel)
        splitter.addWidget(center_panel)
        splitter.addWidget(right_panel)
        splitter.setSizes([290, 790, 440])
        self.setCentralWidget(splitter)
        self.apply_style()
        self.refresh_projects()

    def make_mode_button(self, text: str, mode: str) -> QPushButton:
        btn = QPushButton(text)
        btn.setCheckable(True)
        btn.clicked.connect(lambda: self.set_mode(mode))
        return btn

    def make_model_combo(self, default_text: str) -> QComboBox:
        combo = QComboBox()
        combo.setEditable(True)
        combo.addItem(default_text)
        combo.setCurrentText(default_text)
        return combo

    def combo_text(self, combo: QComboBox) -> str:
        return combo.currentText().strip()

    def set_mode(self, mode: str):
        self.current_mode = mode
        self.basic_mode_btn.setChecked(mode == "basic")
        self.critic_mode_btn.setChecked(mode == "critic")
        self.chain_mode_btn.setChecked(mode == "chain")
        self.overview_mode_btn.setChecked(mode == "overview")
        self.compare_mode_btn.setChecked(mode == "compare")
        names = {"basic": "Обычный RAG", "critic": "RAG + критик", "chain": "Цепочка агентов", "overview": "Обзор проекта", "compare": "Сравнение документов"}
        if mode == "overview":
            self.question_input.setPlaceholderText("Можно оставить пустым и нажать «Спросить» для обзора проекта")
        elif mode == "compare":
            self.question_input.setPlaceholderText("Выдели 2+ документа слева. Можно оставить пустым для общего сравнения")
        else:
            self.question_input.setPlaceholderText("Задай вопрос по документам...")
        self.log(f"Выбран режим: {names.get(mode, mode)}")

    def apply_style(self):
        self.setStyleSheet("""
            QMainWindow { background: #151515; }
            QWidget { background: #151515; color: #eeeeee; font-size: 14px; }
            QTextEdit, QLineEdit, QListWidget, QComboBox, QSpinBox, QDoubleSpinBox {
                background: #202020; color: #f2f2f2; border: 1px solid #3a3a3a; border-radius: 8px; padding: 8px; selection-background-color: #555555;
            }
            QPushButton { background: #2d2d2d; color: #ffffff; border: 1px solid #444444; border-radius: 8px; padding: 8px; }
            QPushButton:hover { background: #3a3a3a; }
            QPushButton:pressed { background: #4a4a4a; }
            QPushButton:checked { background: #505050; border: 1px solid #777777; }
            QGroupBox { border: 1px solid #3a3a3a; border-radius: 8px; margin-top: 10px; padding: 10px; }
            QGroupBox::title { subcontrol-origin: margin; left: 10px; padding: 0 4px; }
            QLabel { color: #dddddd; font-weight: 600; }
        """)

    def log(self, text: str):
        self.log_view.append(text)

    def refresh_projects(self):
        self.project_list.clear()
        self.projects = self.store.list_projects()
        for project in self.projects:
            self.project_list.addItem(project.name)
        if self.projects and self.current_project is None:
            self.project_list.setCurrentRow(0)

    def on_project_selected(self, row: int):
        if row < 0 or row >= len(self.projects):
            return
        self.current_project = self.projects[row]
        self.chat_view.clear()
        self.chat_view.append(f"Открыт проект: {self.current_project.name}\n")
        self.refresh_docs_view()

    def refresh_docs_view(self):
        self._updating_docs_list = True
        self.docs_list.clear()

        if not self.current_project:
            self.docs_info_view.setText("Проект не выбран.")
            self._updating_docs_list = False
            return

        files = self.store.list_document_files(self.current_project)
        state = self.store.sync_document_state(self.current_project)
        chunks = self.store.load_chunks(self.current_project)
        active_names = self.store.active_document_names(self.current_project)
        active_chunks = [chunk for chunk in chunks if chunk.source_file in active_names]
        report = self.store.load_index_report(self.current_project)
        summaries = self.store.load_file_summaries(self.current_project)

        for file in files:
            item = QListWidgetItem(file.name)
            item.setFlags(item.flags() | Qt.ItemIsUserCheckable | Qt.ItemIsSelectable | Qt.ItemIsEnabled)
            item.setData(Qt.UserRole, file.name)
            item.setCheckState(Qt.Checked if state.get(file.name, True) else Qt.Unchecked)
            self.docs_list.addItem(item)

        active_count = sum(1 for p in files if state.get(p.name, True))
        inactive_count = len(files) - active_count

        lines = [
            f"Документов: {len(files)}",
            f"Активных: {active_count}",
            f"Отключённых: {inactive_count}",
            f"Фрагментов в индексе: {len(chunks)}",
            f"Активных фрагментов для RAG: {len(active_chunks)}",
        ]

        if summaries.get("files"):
            lines.append(f"Авто-резюме файлов: {len(summaries.get('files', {}))}")

        if summaries.get("overall"):
            lines.append("")
            lines.append("Краткое общее резюме:")
            lines.append(make_snippet(summaries.get("overall", ""), 700))

        if report:
            lines.append("")
            lines.append("Последний отчёт индексации:")
            lines.extend(report[:8])

        if not files:
            lines.append("")
            lines.append("Документы ещё не добавлены.")

        self.docs_info_view.setText("\n".join(lines))
        self._updating_docs_list = False

    def get_selected_document_names(self) -> List[str]:
        names: List[str] = []
        for item in self.docs_list.selectedItems():
            name = item.data(Qt.UserRole)
            if name:
                names.append(str(name))
        return names

    def on_document_check_changed(self, item: QListWidgetItem):
        if self._updating_docs_list or not self.current_project:
            return
        filename = item.data(Qt.UserRole)
        if not filename:
            return
        active = item.checkState() == Qt.Checked
        self.store.set_document_active(self.current_project, str(filename), active)
        self.log(f"Документ {'включён' if active else 'отключён'}: {filename}")
        self.refresh_docs_view()

    def delete_current_project(self):
        if not self.current_project:
            QMessageBox.warning(self, "Нет проекта", "Сначала выбери проект.")
            return
        reply = QMessageBox.question(
            self,
            "Удалить проект",
            f"Удалить проект «{self.current_project.name}» со всеми документами, индексом и резюме?",
            QMessageBox.Yes | QMessageBox.No,
            QMessageBox.No,
        )
        if reply != QMessageBox.Yes:
            return
        project_name = self.current_project.name
        self.store.delete_project(self.current_project)
        self.current_project = None
        self.chat_view.clear()
        self.sources_view.clear()
        self.docs_list.clear()
        self.docs_info_view.clear()
        self.refresh_projects()
        self.log(f"Удалён проект: {project_name}")

    def delete_selected_documents(self):
        if not self.current_project:
            QMessageBox.warning(self, "Нет проекта", "Сначала выбери проект.")
            return
        selected = self.get_selected_document_names()
        if not selected:
            QMessageBox.information(
                self,
                "Документы не выбраны",
                "Выдели один или несколько документов в списке. Галочка включает/отключает документ, а выделение нужно для удаления.",
            )
            return
        reply = QMessageBox.question(
            self,
            "Удалить документы",
            "Удалить выбранные документы из проекта?\n\n" + "\n".join(selected),
            QMessageBox.Yes | QMessageBox.No,
            QMessageBox.No,
        )
        if reply != QMessageBox.Yes:
            return
        self.store.delete_documents(self.current_project, selected)
        self.log(f"Удалено документов: {len(selected)}")
        self.refresh_docs_view()
        self.sources_view.setText(
            "Документы удалены. Старые фрагменты этих документов убраны из индекса. "
            "Для полной пересборки индекса можно нажать «Переиндексировать»."
        )

    def create_project(self):
        name, ok = QInputDialog.getText(self, "Новый проект", "Название проекта:")
        if not ok or not name.strip():
            return
        project = self.store.create_project(name.strip())
        self.current_project = project
        self.refresh_projects()
        for i, p in enumerate(self.projects):
            if p.id == project.id:
                self.project_list.setCurrentRow(i)
                break
        self.log(f"Создан проект: {project.name}")

    def add_documents(self):
        if not self.current_project:
            QMessageBox.warning(self, "Нет проекта", "Сначала создай или выбери проект.")
            return
        file_paths, _ = QFileDialog.getOpenFileNames(self, "Выбери документы", "", "Documents (*.txt *.md *.pdf *.docx *.xlsx *.csv)")
        if not file_paths:
            return
        copied = self.store.import_files(self.current_project, file_paths)
        self.log(f"Добавлено файлов: {len(copied)}")
        self.refresh_docs_view()
        if copied:
            self.start_auto_summary(copied)

    def start_auto_summary(self, copied_files: List[Path]):
        if not self.current_project:
            return
        base_url = self.base_url_input.text().strip()
        chat_model = self.combo_text(self.chat_model_combo)
        if not base_url or not chat_model:
            self.log("Авто-резюме пропущено: не указан LM Studio API или chat model.")
            return
        if chat_model == "local-model":
            self.log("Авто-резюме: проверь/выбери реальную chat-модель, если local-model не является алиасом в LM Studio.")
        self.add_files_btn.setEnabled(False)
        self.project_summary_btn.setEnabled(False)
        self.log("Делаю краткое авто-резюме загруженных файлов...")
        self.sources_view.setText("Авто-резюме загруженных файлов создаётся...")
        self.summary_worker = AutoSummaryWorker(
            self.current_project,
            [str(p) for p in copied_files],
            base_url,
            chat_model,
        )
        self.summary_worker.finished_ok.connect(self.on_auto_summary_finished)
        self.summary_worker.failed.connect(self.on_auto_summary_failed)
        self.summary_worker.start()

    def summarize_project_files(self):
        if not self.current_project:
            QMessageBox.warning(self, "Нет проекта", "Сначала создай или выбери проект.")
            return
        files = self.store.list_document_files(self.current_project, active_only=True)
        if not files:
            QMessageBox.information(self, "Нет файлов", "В проекте пока нет активных документов. Включи документы галочками или добавь новые.")
            return
        self.start_auto_summary(files)

    def on_auto_summary_finished(self, data: Dict[str, Any]):
        self.add_files_btn.setEnabled(True)
        self.project_summary_btn.setEnabled(True)
        self.log(data.get("message", "Авто-резюме готово."))
        for line in data.get("report", []):
            self.log(line)
        self.sources_view.setText(data.get("summary_text", ""))
        self.refresh_docs_view()

    def on_auto_summary_failed(self, error: str):
        self.add_files_btn.setEnabled(True)
        self.project_summary_btn.setEnabled(True)
        self.log(f"Авто-резюме не удалось: {error}")
        QMessageBox.warning(
            self,
            "Авто-резюме",
            "Файлы добавлены, но краткое резюме создать не удалось.\n\n"
            f"{error}\n\n"
            "Проверь LM Studio и chat-модель. Индексацию можно выполнить отдельно."
        )

    def update_combo_models(self, combo: QComboBox, models: List[str], preferred: Optional[str] = None):
        current = combo.currentText().strip()
        combo.blockSignals(True)
        combo.clear()
        for model in models:
            combo.addItem(model)
        target = preferred or current
        if target and combo.findText(target) < 0:
            combo.insertItem(0, target)
        if target:
            combo.setCurrentText(target)
        elif models:
            combo.setCurrentIndex(0)
        combo.blockSignals(False)

    def choose_default_models(self, models: List[str]) -> Dict[str, str]:
        if not models:
            return {}
        embed_keywords = ("embed", "embedding", "bge", "nomic", "e5", "gte")
        embedding_candidates = [m for m in models if any(k in m.lower() for k in embed_keywords)]
        chat_candidates = [m for m in models if m not in embedding_candidates]
        return {"chat": chat_candidates[0] if chat_candidates else models[0], "embedding": embedding_candidates[0] if embedding_candidates else models[0]}

    def check_lmstudio(self):
        base_url = self.base_url_input.text().strip()
        chat_model = self.combo_text(self.chat_model_combo)
        embedding_model = self.combo_text(self.embedding_model_combo)
        if not base_url:
            QMessageBox.warning(self, "Настройки", "Укажи LM Studio API.")
            return
        self.check_btn.setEnabled(False)
        self.log("Проверяю LM Studio...")
        self.diagnostic_worker = DiagnosticWorker(base_url, chat_model, embedding_model)
        self.diagnostic_worker.finished_ok.connect(self.on_diagnostic_finished)
        self.diagnostic_worker.failed.connect(self.on_worker_failed)
        self.diagnostic_worker.start()

    def on_diagnostic_finished(self, data: Dict[str, Any]):
        self.check_btn.setEnabled(True)
        models = data.get("models", [])
        report = data.get("report", "")
        defaults = self.choose_default_models(models)
        current_chat = self.combo_text(self.chat_model_combo)
        current_embed = self.combo_text(self.embedding_model_combo)
        chat_default = defaults.get("chat") if current_chat in {"", "local-model"} else current_chat
        embed_default = defaults.get("embedding") if current_embed in {"", "text-embedding-nomic-embed-text-v1.5"} else current_embed
        self.update_combo_models(self.chat_model_combo, models, preferred=chat_default)
        self.update_combo_models(self.rewriter_model_combo, models, preferred=chat_default)
        self.update_combo_models(self.critic_model_combo, models, preferred=chat_default)
        self.update_combo_models(self.embedding_model_combo, models, preferred=embed_default)
        self.log(report)
        QMessageBox.information(self, "Проверка LM Studio", report)

    def reindex_project(self):
        if not self.current_project:
            QMessageBox.warning(self, "Нет проекта", "Сначала создай или выбери проект.")
            return
        base_url = self.base_url_input.text().strip()
        embedding_model = self.combo_text(self.embedding_model_combo)
        if not base_url or not embedding_model:
            QMessageBox.warning(self, "Настройки", "Укажи LM Studio API и embedding model.")
            return
        chunk_max_chars = self.chunk_size_input.value()
        chunk_overlap = self.chunk_overlap_input.value()

        if chunk_overlap > chunk_max_chars // 2:
            QMessageBox.warning(
                self,
                "Настройки chunks",
                "Overlap не должен быть больше половины chunk size. Уменьши overlap."
            )
            return

        self.index_btn.setEnabled(False)
        self.log(
            f"Индексация началась... chunk_size={chunk_max_chars}, overlap={chunk_overlap}"
        )
        self.index_worker = IndexWorker(
            self.current_project,
            base_url,
            embedding_model,
            chunk_max_chars,
            chunk_overlap,
        )
        self.index_worker.finished_ok.connect(self.on_index_finished)
        self.index_worker.failed.connect(self.on_worker_failed)
        self.index_worker.start()

    def on_index_finished(self, data: Dict[str, Any]):
        self.index_btn.setEnabled(True)
        message = data.get("message", "Индексация завершена.")
        report = data.get("report", [])
        self.log(message)
        for line in report:
            self.log(line)
        self.sources_view.setText("Отчёт индексации:\n" + "\n".join(report))
        self.refresh_docs_view()

    def ask_question(self):
        if not self.current_project:
            QMessageBox.warning(self, "Нет проекта", "Сначала создай или выбери проект.")
            return
        question = self.question_input.text().strip()
        if not question and self.current_mode not in {"overview", "compare"}:
            return
        if self.current_mode == "overview" and not question:
            question = "Сделай обзор проекта"
        if self.current_mode == "compare" and not question:
            question = "Сравни выбранные документы"
        base_url = self.base_url_input.text().strip()
        chat_model = self.combo_text(self.chat_model_combo)
        rewriter_model = self.combo_text(self.rewriter_model_combo) or chat_model
        critic_model = self.combo_text(self.critic_model_combo) or chat_model
        embedding_model = self.combo_text(self.embedding_model_combo)
        top_k = self.top_k_input.value()
        min_score = self.min_score_input.value()
        if not base_url or not chat_model or not embedding_model:
            QMessageBox.warning(self, "Настройки", "Укажи API, Answer model и Embedding model.")
            return
        mode_names = {"basic": "Обычный RAG", "critic": "RAG + критик", "chain": "Цепочка агентов", "overview": "Обзор проекта", "compare": "Сравнение документов"}
        mode_name = mode_names.get(self.current_mode, self.current_mode)
        self.question_input.clear()
        self.ask_btn.setEnabled(False)
        self.sources_view.clear()
        self.chat_view.append(f"\nТы: {question}\n")
        self.chat_view.append(f"Режим: {mode_name}\n")
        self.chat_view.append("Ассистент думает...\n")
        selected_documents = self.get_selected_document_names()
        if self.current_mode == "compare" and len(selected_documents) < 2:
            QMessageBox.information(
                self,
                "Сравнение документов",
                "Для сравнения выдели минимум два документа в списке слева. Галочки включают/отключают документы, а выделение мышкой выбирает документы для сравнения.",
            )
            self.ask_btn.setEnabled(True)
            return
        self.chat_worker = ChatWorker(self.current_project, question, base_url, chat_model, embedding_model, rewriter_model, critic_model, self.current_mode, top_k, min_score, selected_documents=selected_documents)
        self.chat_worker.finished_ok.connect(self.on_answer_finished)
        self.chat_worker.failed.connect(self.on_worker_failed)
        self.chat_worker.start()

    def on_answer_finished(self, result: Dict[str, str]):
        self.ask_btn.setEnabled(True)
        answer = result.get("answer", "")
        debug = result.get("debug", "")
        self.chat_view.append(f"Ассистент:\n{answer}\n")
        self.sources_view.setText(debug)
        self.chat_view.verticalScrollBar().setValue(self.chat_view.verticalScrollBar().maximum())

    def on_worker_failed(self, error: str):
        self.index_btn.setEnabled(True)
        self.ask_btn.setEnabled(True)
        self.check_btn.setEnabled(True)
        self.project_summary_btn.setEnabled(True)
        self.add_files_btn.setEnabled(True)
        self.log(f"Ошибка: {error}")
        QMessageBox.critical(self, "Ошибка", f"Что-то пошло не так.\n\n{error}\n\nПроверь, что LM Studio запущен, сервер включён, а названия моделей указаны правильно.")


def main():
    ensure_dirs()
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec())


if __name__ == "__main__":
    main()