settings.py - master - zimaaadev/Parser

settings.py
95 строк3 KB
zimaaadev
Ищет природные объекты, набнрежные и т.п.
24 дек 2025, 17:18
24 дек 2025, 17:182d73bc2
# parsers/settings.py
import os
from dotenv import load_dotenv

load_dotenv()

def load_user_agents():
    agents = []
    i = 1
    while True:
        agent = os.getenv(f"USER_AGENT_{i}")
        if not agent:
            break
        agents.append(agent.strip())
        i += 1
    return agents or [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    ]

def load_proxies():
    proxies = []
    i = 1
    while True:
        proxy = os.getenv(f"PROXY_{i}")
        if not proxy:
            break
        proxies.append(proxy.strip())
        i += 1
    return proxies

# === Города и категории ===
CITIES = {
    "Майкоп": "maykop",
    "Ростов-на-Дону": "rostov-na-donu",
    "Краснодар": "krasnodar",
}

CATEGORIES = {
    "дома": "dom",
    "квартиры": "kvartira",
    "коммерческая": "kommercheskaya",
}

CIAN_OFFER_TYPES = {
    "квартиры": "flat",
    "дома": "house",
    "коммерческая": "commercial",
}

# === Базовые URL ===
YANDEX_BASE_URL = "https://realty.yandex.ru/{city}/kupit/{category}/bez-posrednikov/"
CIAN_BASE_URL = "https://www.cian.ru/cat.php?deal_type=sale&engine_version=2&offer_type={offer_type}&region=4536&with_salers=0"

# === Таймауты ===
HTTP_TIMEOUT = 30.0
MAX_RETRIES = 3

# === Overpass ===
OVERPASS_ENDPOINTS = [
    "https://overpass.kumi.systems/api/interpreter",
    "https://overpass-api.de/api/interpreter",
    "https://z.overpass-api.de/api/interpreter"
]

# === OSM категории ===
OSM_CATEGORY_MAPPING = {
    ("leisure", "park"): "парк",
    ("leisure", "garden"): "сквер",
    ("natural", "wood"): "лес",
    ("natural", "forest"): "лес",
    ("natural", "water"): "водоём",
    ("waterway", "river"): "река",
    ("waterway", "stream"): "речка",
    ("natural", "lake"): "озеро",
    ("natural", "pond"): "пруд",
    ("leisure", "beach"): "пляж",
    ("man_made", "pier"): "набережная",
    ("tourism", "museum"): "достопримечательность",
    ("historic", "church"): "достопримечательность",
    ("amenity", "pharmacy"): "аптека",
    ("shop", "supermarket"): "супермаркет",
}

# === Прокси и UA ===
USER_AGENTS = load_user_agents()
PROXIES_LIST = load_proxies()

# settings.py
PROXIES = {
    "list": [],
    "user_agents": [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ..."
    ]
}