Backend/search_engine.py - master - Alekron/Unbeatable_Benchmarking

Backend/search_engine.py
192 строки6 KB
Alekron
Initial Backend
23 ноя 2025, 09:12
23 ноя 2025, 09:12245fffc
import requests
from bs4 import BeautifulSoup
import logging
from typing import List
import time
import random
from fake_useragent import UserAgent
import re

logger = logging.getLogger(__name__)


class SearchEngine:
    def __init__(self):
        self.ua = UserAgent()
        self.session = requests.Session()

    def get_headers(self):
        return {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3',
        }

    def search_simple(self, query: str, num_results: int = 5) -> List[str]:
        """Простой поиск через прямой запрос к поисковикам"""
        try:
            search_engines = [
                self._try_duckduckgo_simple,
                self._try_google_direct,
                self._try_bing_direct
            ]

            for search_func in search_engines:
                urls = search_func(query, num_results)
                if urls:
                    logger.info(f"Найдено {len(urls)} URLs используя {search_func.__name__}")
                    return urls

            return []

        except Exception as e:
            logger.error(f"Поисковая ошибка: {e}")
            return []

    def _try_duckduckgo_simple(self, query: str, num_results: int) -> List[str]:
        """Упрощенный DuckDuckGo"""
        try:
            search_url = "https://html.duckduckgo.com/html/"
            data = {
                'q': query,
                'b': ''
            }

            response = self.session.post(
                search_url,
                data=data,
                headers=self.get_headers(),
                timeout=15
            )

            if response.status_code != 200:
                return []

            soup = BeautifulSoup(response.content, 'html.parser')
            urls = []

            for result in soup.select('.result__url'):
                url_text = result.get_text(strip=True)
                if url_text:
                    full_url = f"https://{url_text}" if not url_text.startswith('http') else url_text
                    urls.append(full_url)
                    if len(urls) >= num_results:
                        break

            return urls

        except Exception as e:
            logger.error(f"DuckDuckGo simple error: {e}")
            return []

    def _try_google_direct(self, query: str, num_results: int) -> List[str]:
        """Прямой поиск Google"""
        try:
            search_url = "https://www.google.com/search"
            params = {
                'q': query,
                'num': num_results + 2
            }

            response = self.session.get(
                search_url,
                params=params,
                headers=self.get_headers(),
                timeout=15
            )

            if response.status_code != 200:
                return []

            soup = BeautifulSoup(response.content, 'html.parser')
            urls = []

            for link in soup.find_all('a', href=re.compile(r'^/url\?q=')):
                href = link.get('href')
                if href:
                    match = re.search(r'/url\?q=(https?://[^&]+)', href)
                    if match:
                        url = match.group(1)
                        urls.append(url)
                        if len(urls) >= num_results:
                            break

            return urls

        except Exception as e:
            logger.error(f"Google ошибка: {e}")
            return []

    def _try_bing_direct(self, query: str, num_results: int) -> List[str]:
        """Прямой поиск Bing"""
        try:
            search_url = "https://www.bing.com/search"
            params = {
                'q': query,
                'count': num_results
            }

            response = self.session.get(
                search_url,
                params=params,
                headers=self.get_headers(),
                timeout=15
            )

            if response.status_code != 200:
                return []

            soup = BeautifulSoup(response.content, 'html.parser')
            urls = []

            for link in soup.find_all('a', href=re.compile(r'^http')):
                href = link.get('href')
                if href:
                    urls.append(href)
                    if len(urls) >= num_results:
                        break

            return urls

        except Exception as e:
            logger.error(f"Bing ошибка: {e}")
            return []

    def get_page_content(self, url: str) -> str:
        """Получает контент страницы"""
        try:
            time.sleep(random.uniform(1, 2))

            response = self.session.get(
                url,
                headers=self.get_headers(),
                timeout=20,
                allow_redirects=True
            )

            if response.status_code != 200:
                return ""

            for encoding in ['utf-8', 'windows-1251', 'cp1251']:
                try:
                    response.encoding = encoding
                    soup = BeautifulSoup(response.content, 'html.parser')
                    break
                except:
                    continue

            for element in soup(["script", "style", "nav", "footer", "header", "aside"]):
                element.decompose()

            text = soup.get_text()

            lines = (line.strip() for line in text.splitlines())
            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
            clean_text = ' '.join(chunk for chunk in chunks if chunk)

            logger.info(f"Extracted {len(clean_text)} chars from {url}")
            return clean_text[:10000] if len(clean_text) > 200 else ""

        except Exception as e:
            logger.error(f"Ошибка парсинга{url}: {e}")
            return ""