app/utils/text_utils.py - master - artyuhovs/AIFeedBackTrainingBot

app/utils/text_utils.py
114 строк3 KB
Codex
Завершить архитектурное оздоровление и исправить итоги
15 июл 2026, 11:05
15 июл 2026, 11:05c65f9a6
from __future__ import annotations

import re
from dataclasses import dataclass

TELEGRAM_MAX_TEXT = 4096

_WORD_PATTERN = re.compile(r"(?u)[0-9A-Za-zА-Яа-яЁё]+(?:[-'][0-9A-Za-zА-Яа-яЁё]+)*")

_FILLER_PATTERN = re.compile(
    r"(?iu)"
    r"(?:\s*[,;:–—-]\s*)?"
    r"(?<![\w-])"
    r"(?:"
    r"как\s+бы|"
    r"так\s+сказать|"
    r"скажем\s+так|"
    r"это\s+самое|"
    r"в\s+общем(?:\s*-?\s*то)?|"
    r"в\s+принципе|"
    r"э+(?:-э+)*|"
    r"э?м+|"
    r"ну|"
    r"вот|"
    r"типа|"
    r"значит|"
    r"короче|"
    r"собственно"
    r")"
    r"(?![\w-])"
    r"(?:\s*[,;:–—-]\s*)?"
)


@dataclass(frozen=True)
class TextStats:
    words: int
    characters: int


def text_stats(text: str) -> TextStats:
    stripped = text.strip()
    return TextStats(
        words=len(_WORD_PATTERN.findall(stripped)),
        characters=len(stripped),
    )


def split_telegram_text(text: str, limit: int = TELEGRAM_MAX_TEXT) -> list[str]:
    if limit <= 0:
        raise ValueError("limit must be positive")
    if _utf16_units(text) <= limit:
        return [text]

    chunks: list[str] = []
    remaining = text
    while remaining:
        max_end = _prefix_end_by_utf16_units(remaining, limit)
        if max_end == len(remaining):
            chunks.append(remaining)
            break
        if max_end == 0:
            raise ValueError("limit is too small for a single character")
        cut = remaining.rfind("\n", 0, max_end)
        if cut < 0 or _utf16_units(remaining[:cut]) < limit // 2:
            cut = remaining.rfind(" ", 0, max_end)
        if cut < 0 or _utf16_units(remaining[:cut]) < limit // 2:
            cut = max_end
        chunks.append(remaining[:cut].rstrip())
        remaining = remaining[cut:].lstrip()
    return chunks


def _utf16_units(text: str) -> int:
    """Return the length Telegram uses for text/entity offsets."""
    return sum(2 if ord(character) > 0xFFFF else 1 for character in text)


def _prefix_end_by_utf16_units(text: str, limit: int) -> int:
    units = 0
    for index, character in enumerate(text):
        character_units = 2 if ord(character) > 0xFFFF else 1
        if units + character_units > limit:
            return index
        units += character_units
    return len(text)


def remove_filler_words(text: str) -> str:
    """Remove common filler words without rewriting or adding spoken content."""
    cleaned = text.strip()
    previous = None
    while cleaned != previous:
        previous = cleaned
        cleaned = _FILLER_PATTERN.sub(" ", cleaned)
    return _capitalize_sentence_starts(_normalize_after_filler_removal(cleaned))


def _normalize_after_filler_removal(text: str) -> str:
    text = re.sub(r"\s+", " ", text)
    text = re.sub(r"\s+([,.;:!?])", r"\1", text)
    text = re.sub(r"([.!?])\s*[,;:]+\s*", r"\1 ", text)
    text = re.sub(r"[,;:]\s*([.!?])", r"\1", text)
    text = re.sub(r"^[\s,;:–—-]+", "", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def _capitalize_sentence_starts(text: str) -> str:
    return re.sub(
        r"(^|[.!?]\s+)([a-zа-яё])",
        lambda match: match.group(1) + match.group(2).upper(),
        text,
    )