/
zimaaadev
/
Parser
Обзор
Документация
Войти
/
zimaaadev
/
Parser
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
admin/app.py
741 строка
28 KB
zimaaa
Добавил страницу Logs - отображение логов в админке
11 дек 2025, 21:12
11 дек 2025, 21:12
6f30592
Код
Авторство
О чём код?
# admin/app.py from flask import Flask, render_template, request, redirect, url_for, session, flash, jsonify, make_response from werkzeug.security import check_password_hash, generate_password_hash import sqlite3 import os import subprocess import sys from datetime import datetime, timedelta from dotenv import load_dotenv, set_key, dotenv_values import json from math import ceil import csv from io import StringIO import threading import logging from logging.handlers import RotatingFileHandler # === Загрузка переменных окружения === load_dotenv() # Настройка логирования handler = RotatingFileHandler('app.log', maxBytes=10*1024*1024, backupCount=5) handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) app.logger.addHandler(handler) app.logger.setLevel(logging.INFO) # === Основные пути === PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATA_DIR = os.path.join(PROJECT_DIR, "data") os.makedirs(DATA_DIR, exist_ok=True) # Читаем путь из .env или используем realty.db по умолчанию env_db_path = os.getenv("DATABASE_PATH") if env_db_path: if env_db_path.startswith("./") or env_db_path.startswith(".\\"): env_db_path = os.path.join(PROJECT_DIR, os.path.relpath(env_db_path, ".")) DATABASE_PATH = os.path.abspath(env_db_path) else: DATABASE_PATH = os.path.join(DATA_DIR, "realty.db") DATABASE_PATH = os.path.abspath(DATABASE_PATH) print(f"✅ Админка использует БД: {DATABASE_PATH}") # === Пути к конфигурационным файлам === DOTENV_PATH = os.path.join(PROJECT_DIR, ".env") MANUAL_LOCATIONS_FILE = os.path.join(DATA_DIR, "manual_locations.json") # Загружаем переменные из .env HEALTH_TOKEN = os.getenv("HEALTH_CHECK_TOKEN", "secret-health-token") app = Flask(__name__) app.secret_key = os.getenv("FLASK_SECRET_KEY", "super-secret-key-change-in-production") PER_PAGE = 20 # Количество дней для "новых" объявлений NEW_PERIOD_DAYS = 1 # === Кастомные фильтры Jinja2 === @app.template_filter('format_number') def format_number(value): if value is None: return "" return f"{value:,}".replace(",", " ") @app.template_filter('tojson') def tojson_filter(obj): return json.dumps(obj, ensure_ascii=False, default=str) # === Обновление из GIT через админку === @app.route('/api/update', methods=['POST']) @login_required def update_system(): try: data = request.get_json() branch = data.get('branch', 'main').strip() git_username = data.get('username', '').strip() git_password = data.get('password', '').strip() # Проверка, что это Git-репозиторий if not os.path.exists('.git'): return jsonify({"success": False, "output": "❌ Это не Git-репозиторий!"}) # Сохраняем .env if os.path.exists('.env'): os.system('cp .env /tmp/.env.backup') output = f"🔄 Обновление из ветки: {branch}\n" # Если переданы логин и пароль — подставляем в URL repo_url = subprocess.check_output(['git', 'config', '--get', 'remote.origin.url'], text=True).strip() if git_username and git_password: # Заменяем https:// на https://user:pass@ if repo_url.startswith("https://"): repo_host = repo_url.split("://")[1] repo_url = f"https://{git_username}:{git_password}@{repo_host}" output += "🔐 Используется аутентификация через логин/пароль\n" else: output += "⚠️ Аутентификация поддерживается только для HTTPS\n" # Выполняем pull try: subprocess.run(['git', 'fetch', repo_url, branch], check=True, capture_output=True, text=True) subprocess.run(['git', 'reset', '--hard', f'origin/{branch}'], check=True, capture_output=True, text=True) output += f"✅ Ветка '{branch}' успешно обновлена!\n" except subprocess.CalledProcessError as e: return jsonify({"success": False, "output": output + f"❌ Ошибка Git:\n{e.stderr or e.stdout}\n"}) # Восстанавливаем .env if os.path.exists('/tmp/.env.backup'): os.system('cp /tmp/.env.backup .env') # Устанавливаем зависимости output += "📦 Устанавливаем зависимости...\n" pip_result = subprocess.run( [os.path.join('venv', 'bin', 'pip'), 'install', '-r', 'requirements.txt'], capture_output=True, text=True ) if pip_result.returncode == 0: output += "✅ Зависимости установлены.\n" else: output += "⚠️ Ошибка pip:\n" + pip_result.stdout + pip_result.stderr + "\n" # Перезапускаем сервисы try: subprocess.run(['sudo', 'systemctl', 'restart', 'admin-panel'], check=True) subprocess.run(['sudo', 'systemctl', 'restart', 'realty-parser'], check=True) output += "✅ Сервисы перезапущены.\n" except subprocess.CalledProcessError as e: output += f"❌ Ошибка перезапуска сервисов: {e}\n" return jsonify({"success": True, "output": output}) except Exception as e: return jsonify({"success": False, "output": f"❌ Ошибка: {str(e)}"}) # === Получение статистики для дашборда === def get_dashboard_stats(): conn = get_db_connection() try: # Общее количество total = conn.execute("SELECT COUNT(*) as cnt FROM items").fetchone()["cnt"] # Средняя цена avg_price_row = conn.execute("SELECT AVG(price) as avg FROM items").fetchone() avg_price = int(avg_price_row["avg"]) if avg_price_row["avg"] else 0 # Новые за последние сутки new_count = conn.execute(""" SELECT COUNT(*) as cnt FROM items WHERE parsed_at >= DATE('now', '-%d days') """ % NEW_PERIOD_DAYS).fetchone()["cnt"] # Последний запуск last_parsed = conn.execute(""" SELECT MAX(parsed_at) as last FROM items """).fetchone()["last"] # Количество районов districts_count = conn.execute(""" SELECT COUNT(DISTINCT district) as cnt FROM items WHERE district IS NOT NULL AND TRIM(district) != '' AND district != 'Не определено' """).fetchone()["cnt"] # Мин/макс цена min_max = conn.execute(""" SELECT MIN(price) as min, MAX(price) as max FROM items """).fetchone() min_price = min_max["min"] or 0 max_price = max_price = min_max["max"] or 0 return { "total": total, "avg_price": avg_price, "new_count": new_count, "last_parsed": last_parsed, "districts_count": districts_count, "min_price": min_price, "max_price": max_price } except Exception as e: print(f"❌ Ошибка статистики: {e}") return { "total": 0, "avg_price": 0, "new_count": 0, "last_parsed": None, "districts_count": 0, "min_price": 0, "max_price": 0 } finally: conn.close() @app.template_filter('paginate_url') def paginate_url_filter(args, page_num): args_dict = args.to_dict() args_dict.pop('page', None) return url_for('index', page=page_num, **args_dict) # === Утилиты для .env === def get_env_var(key: str) -> str: return dotenv_values(DOTENV_PATH).get(key) def update_dotenv(key: str, value: str): set_key(DOTENV_PATH, key, value) os.environ[key] = value # Обновляем текущую сессию # === Админ: логин === ADMIN_LOGIN = os.getenv("ADMIN_LOGIN", "admin") def get_admin_password_hashed(): return get_env_var("ADMIN_PASSWORD_HASHED") @app.route("/register", methods=["GET", "POST"]) def register(): if get_admin_password_hashed(): return redirect(url_for("login")) if request.method == "POST": username = request.form["username"].strip() password = request.form["password"] confirm = request.form["confirm"] if not username: flash("Введите логин", "danger") elif len(password) < 4: flash("Пароль должен быть не менее 4 символов", "danger") elif password != confirm: flash("Пароли не совпадают", "danger") else: # ✅ Исправлено: явно указан pbkdf2:sha256 hashed = generate_password_hash(password, method='pbkdf2:sha256') update_dotenv("ADMIN_LOGIN", username) update_dotenv("ADMIN_PASSWORD_HASHED", hashed) flash("✅ Администратор зарегистрирован! Войдите.", "success") return redirect(url_for("login")) return render_template("register.html") # === Инициализация базы данных === def init_db(): with sqlite3.connect(DATABASE_PATH) as conn: conn.execute(""" CREATE TABLE IF NOT EXISTS items ( id INTEGER PRIMARY KEY AUTOINCREMENT, avito_id TEXT UNIQUE NOT NULL, title TEXT NOT NULL, price INTEGER, price_per_m2 TEXT, url TEXT NOT NULL, location TEXT, street TEXT, house TEXT, description TEXT, published_at TEXT, seller TEXT, house_type TEXT, sale_type TEXT, price_negotiable INTEGER DEFAULT 0, documents_verified INTEGER DEFAULT 0, image_url TEXT, is_new INTEGER DEFAULT 0, district TEXT, lat REAL, lng REAL, parsed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """) # Индексы для быстрого поиска conn.execute("CREATE INDEX IF NOT EXISTS idx_district ON items(district);") conn.execute("CREATE INDEX IF NOT EXISTS idx_price ON items(price);") conn.execute("CREATE INDEX IF NOT EXISTS idx_parsed_at ON items(parsed_at DESC);") conn.execute("CREATE INDEX IF NOT EXISTS idx_avito_id ON items(avito_id);") conn.execute("CREATE INDEX IF NOT EXISTS idx_seller ON items(seller);") conn.execute("CREATE INDEX IF NOT EXISTS idx_is_new ON items(is_new);") print(f"✅ Админка инициализировала базу: {DATABASE_PATH}") def get_db_connection(): conn = sqlite3.connect(DATABASE_PATH) conn.row_factory = sqlite3.Row # Доступ к колонкам по имени return conn # === Аутентификация === def login_required(f): from functools import wraps @wraps(f) def decorated_function(*args, **kwargs): if not session.get('logged_in'): flash("Пожалуйста, войдите для доступа.", "info") return redirect(url_for('login')) return f(*args, **kwargs) return decorated_function # === Страница логами в админке === @app.route('/logs') @login_required def logs(): log_file = 'app.log' if not os.path.exists(log_file): lines = ["❌ Файл логов не найден: app.log"] else: try: with open(log_file, 'r', encoding='utf-8') as f: lines = f.readlines() lines = [line.strip() for line in lines[-100:]] # последние 100 строк except Exception as e: lines = [f"❌ Ошибка чтения логов: {e}"] return render_template('logs.html', logs=lines) @app.route("/login", methods=["GET", "POST"]) def login(): if not get_admin_password_hashed(): return redirect(url_for("register")) if request.method == "POST": username = request.form["username"] password = request.form["password"] stored_hash = get_admin_password_hashed() if username == get_env_var("ADMIN_LOGIN") and stored_hash and check_password_hash(stored_hash, password): session['logged_in'] = True flash("✅ Успешный вход!", "success") return redirect(url_for("index")) else: flash("❌ Неверный логин или пароль", "danger") return render_template("login.html") @app.route("/logout") def logout(): session.clear() flash("Вы вышли из системы.", "info") return redirect(url_for("login")) # === Главная страница — список объявлений === @app.route("/") @login_required def index(): page = max(1, request.args.get("page", 1, type=int)) title = request.args.get("title", "").strip() min_price = request.args.get("min_price", type=int) max_price = request.args.get("max_price", type=int) location = request.args.get("location", "").strip() district_filter = request.args.get("district", "").strip() period_days = request.args.get("period", type=int) theme = request.cookies.get("theme", "light") offset = (page - 1) * PER_PAGE conn = get_db_connection() db_districts = conn.execute(""" SELECT DISTINCT district FROM items WHERE district IS NOT NULL AND TRIM(district) != '' AND district != 'Не определено' ORDER BY district """).fetchall() ALL_DISTRICTS = [r["district"] for r in db_districts] conn.close() query = """ SELECT id, avito_id, title, price, price_per_m2, url, location, street, house, description, published_at, seller, house_type, sale_type, price_negotiable, documents_verified, image_url, is_new, district, lat, lng, parsed_at FROM items WHERE 1=1 """ count_query = "SELECT COUNT(*) AS cnt FROM items WHERE 1=1" sum_query = "SELECT SUM(price) as total, AVG(price) as avg FROM items WHERE 1=1" params = [] if district_filter and district_filter.lower() != "не определено": query += " AND district = ?" count_query += " AND district = ?" sum_query += " AND district = ?" params.append(district_filter) elif district_filter.lower() == "не определено": query += " AND (district IS NULL OR TRIM(district) = '' OR district = 'Не определено')" count_query += " AND (district IS NULL OR TRIM(district) = '' OR district = 'Не определено')" sum_query += " AND (district IS NULL OR TRIM(district) = '' OR district = 'Не определено')" if title: query += " AND title LIKE ?" count_query += " AND title LIKE ?" sum_query += " AND title LIKE ?" params.append(f"%{title}%") if min_price is not None: query += " AND price >= ?" count_query += " AND price >= ?" sum_query += " AND price >= ?" params.append(min_price) if max_price is not None: query += " AND price <= ?" count_query += " AND price <= ?" sum_query += " AND price <= ?" params.append(max_price) if location: query += " AND location LIKE ?" count_query += " AND location LIKE ?" sum_query += " AND location LIKE ?" params.append(f"%{location}%") if period_days: query += " AND parsed_at >= DATE('now', '-%d days')" % period_days count_query += " AND parsed_at >= DATE('now', '-%d days')" % period_days sum_query += " AND parsed_at >= DATE('now', '-%d days')" % period_days query += " ORDER BY parsed_at DESC LIMIT ? OFFSET ?" params.extend([PER_PAGE, offset]) try: conn = get_db_connection() items = conn.execute(query, params).fetchall() total_items = conn.execute(count_query, params[:-2]).fetchone()["cnt"] sum_row = conn.execute(sum_query, params[:-2]).fetchone() conn.close() except Exception as e: flash(f"❌ Ошибка базы данных: {str(e)}", "danger") items = [] total_items = 0 sum_row = {"total": 0, "avg": 0} total_pages = (total_items + PER_PAGE - 1) // PER_PAGE prev_page = page - 1 if page > 1 else None next_page = page + 1 if page < total_pages else None last_run = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # Статистика цен price_ranges = [0, 0, 0, 0, 0] for item in items: p = item["price"] if p < 3_000_000: price_ranges[0] += 1 elif p < 5_000_000: price_ranges[1] += 1 elif p < 7_000_000: price_ranges[2] += 1 elif p < 10_000_000: price_ranges[3] += 1 else: price_ranges[4] += 1 # Средняя и общая цена avg_price = int(sum_row["avg"]) if sum_row["avg"] else 0 total_price = int(sum_row["total"]) if sum_row["total"] else 0 stats = get_dashboard_stats() return render_template( "index.html", items=items, total_items=total_items, last_run=last_run, page=page, prev_page=prev_page, next_page=next_page, total_pages=total_pages, theme=theme, ALL_DISTRICTS=ALL_DISTRICTS, avg_price=avg_price, total_price=total_price, stats=stats, price_ranges=price_ranges ) @app.route("/clear-old") @login_required def clear_old(): cutoff = datetime.now() - timedelta(days=30) with get_db_connection() as conn: deleted = conn.execute("DELETE FROM items WHERE parsed_at < ?", (cutoff,)).rowcount flash(f"🗑️ Удалено {deleted} старых объявлений", "info") return redirect(url_for("index")) # === Карта === @app.route("/map") @login_required def map_view(): conn = get_db_connection() items = conn.execute(""" SELECT title, price, url, location, district, lat, lng, price_per_m2, house_type, sale_type, image_url FROM items WHERE lat IS NOT NULL AND lng IS NOT NULL ORDER BY parsed_at DESC LIMIT 100 """).fetchall() conn.close() return render_template("map.html", items=items) # === Фоновый запуск парсера === parser_running = False parser_lock = threading.Lock() @app.route("/run-parser", methods=["POST"]) @login_required def run_parser(): global parser_running with parser_lock: if parser_running: flash("🟡 Парсер уже запущен!", "warning") return redirect(url_for("index")) parser_running = True def run(): # Путь к парсеру — от корня проекта parser_script = os.path.join(PROJECT_DIR, "parser", "parser_cian.py") if not os.path.exists(parser_script): print(f"🔴 Файл парсера не найден: {parser_script}") return try: # Запускаем с правильным cwd = корень проекта subprocess.run([ sys.executable, parser_script ], cwd=PROJECT_DIR, check=True) except subprocess.CalledProcessError as e: print(f"🔴 Ошибка выполнения парсера: {e}") except Exception as e: print(f"🔴 Неизвестная ошибка: {e}") finally: with parser_lock: parser_running = False thread = threading.Thread(target=run, daemon=True) thread.start() flash("🔄 Парсинг запущен в фоне...", "info") return redirect(url_for("index")) # === Статистика === @app.route("/stats") @login_required def stats(): conn = get_db_connection() district_data = conn.execute(""" SELECT district, COUNT(*) as count FROM items WHERE district IS NOT NULL AND TRIM(district) != '' AND district != 'Не определено' GROUP BY district ORDER BY count DESC LIMIT 10 """).fetchall() house_type_data = conn.execute(""" SELECT house_type, COUNT(*) as count FROM items WHERE house_type IS NOT NULL AND TRIM(house_type) != '' GROUP BY house_type ORDER BY count DESC LIMIT 10 """).fetchall() conn.close() return render_template( "stats.html", district_labels=[r["district"] for r in district_data], district_values=[r["count"] for r in district_data], house_labels=[r["house_type"] for r in house_type_data], house_values=[r["count"] for r in house_type_data] ) # === Смена темы === @app.route("/set-theme/<theme>", methods=["POST"]) def set_theme(theme): if theme not in ["light", "dark"]: return jsonify({"error": "Invalid theme"}), 400 resp = make_response(jsonify({"status": "ok"})) resp.set_cookie("theme", theme, max_age=30*24*60*60) return resp # === Редактор ручных локаций === def load_manual_locations(): if os.path.exists(MANUAL_LOCATIONS_FILE): try: with open(MANUAL_LOCATIONS_FILE, "r", encoding="utf-8") as f: data = json.load(f) return {k.strip().lower(): v for k, v in data.items()} except Exception as e: print(f"🔴 Ошибка чтения manual_locations.json: {e}") return {} return {} def save_manual_locations(data): try: serializable = {k.strip().lower(): v for k, v in data.items()} with open(MANUAL_LOCATIONS_FILE, "w", encoding="utf-8") as f: json.dump(serializable, f, ensure_ascii=False, indent=2) return True except Exception as e: print(f"🔴 Ошибка записи manual_locations.json: {e}") return False @app.route("/manual-locations", methods=["GET", "POST"]) @login_required def manual_locations(): locations = load_manual_locations() if request.method == "POST": action = request.form.get("action") street = request.form.get("street", "").strip().lower() district = request.form.get("district", "").strip() lat_str = request.form.get("lat", "").strip() lng_str = request.form.get("lng", "").strip() if action == "add": if not all([street, district, lat_str, lng_str]): flash("❌ Все поля обязательны.", "danger") else: try: lat = float(lat_str) lng = float(lng_str) if not (-90 <= lat <= 90): raise ValueError("Широта вне диапазона (-90..90)") if not (-180 <= lng <= 180): raise ValueError("Долгота вне диапазона (-180..180)") locations[street] = [lat, lng, district] if save_manual_locations(locations): flash(f"✅ Улица '{street}' добавлена.", "success") else: flash("❌ Ошибка сохранения файла.", "danger") except ValueError as e: flash(f"❌ Некорректные координаты: {e}", "danger") elif action == "delete": if street in locations: del locations[street] if save_manual_locations(locations): flash(f"✅ Улица '{street}' удалена.", "success") else: flash("❌ Ошибка сохранения файла.", "danger") else: flash("❌ Улица не найдена.", "warning") return redirect(url_for("manual_locations")) return render_template("manual_locations.html", locations=locations) # === Редактирование .env === @app.route("/env", methods=["GET", "POST"]) @login_required def edit_env(): if request.method == "POST": for key, value in request.form.items(): if key.startswith("env_"): env_key = key[4:] update_dotenv(env_key, value.strip()) flash("✅ Файл .env обновлён!", "success") return redirect(url_for("edit_env")) env_vars = {k: v for k, v in dotenv_values(DOTENV_PATH).items()} sensitive_keys = {"ADMIN_PASSWORD_HASHED", "TELEGRAM_BOT_TOKEN", "FLASK_SECRET_KEY", "PROXY_PASS"} for key in sensitive_keys: if key in env_vars: env_vars[key] = "●●●●●●●●" return render_template("env_edit.html", env_vars=env_vars) # === Сброс пароля === @app.route("/reset-password", methods=["GET", "POST"]) @login_required def reset_password(): if request.method == "POST": password = request.form["password"] confirm = request.form["confirm"] if len(password) < 4: flash("Пароль должен быть не менее 4 символов", "danger") elif password != confirm: flash("Пароли не совпадают", "danger") else: hashed = generate_password_hash(password, method='pbkdf2:sha256') update_dotenv("ADMIN_PASSWORD_HASHED", hashed) flash("✅ Пароль изменён!", "success") return redirect(url_for("edit_env")) return render_template("reset_password.html") # === Health check === @app.route("/health") def health(): auth = request.headers.get("Authorization") if not auth or not auth.startswith("Bearer ") or auth.split(" ")[1] != HEALTH_TOKEN: return jsonify({"error": "Unauthorized"}), 401 try: conn = get_db_connection() conn.execute("SELECT 1").fetchone() db_ok = True conn.close() except Exception as e: db_ok = False status = "ok" if db_ok else "error" return jsonify({ "status": status, "timestamp": datetime.now().isoformat(), "database": "connected" if db_ok else "disconnected", "service": "realty-parser-admin" }), 200 if db_ok else 500 # === Экспорт CSV === @app.route("/export.csv") @login_required def export_csv(): conn = get_db_connection() items = conn.execute(""" SELECT avito_id, title, price, price_per_m2, url, location, district, published_at, seller, house_type, sale_type, parsed_at FROM items ORDER BY parsed_at DESC """).fetchall() conn.close() si = StringIO() writer = csv.writer(si, delimiter=";", quoting=csv.QUOTE_ALL) writer.writerow([ "ID Avito", "Название", "Цена", "Цена за м²", "Ссылка", "Адрес", "Район", "Дата публикации", "Продавец", "Тип дома", "Тип сделки", "Дата парсинга" ]) for item in items: writer.writerow([ item["avito_id"], item["title"], item["price"], item["price_per_m2"], item["url"], item["location"], item["district"], item["published_at"], item["seller"], item["house_type"], item["sale_type"], item["parsed_at"] ]) output = si.getvalue() resp = make_response(output.encode('utf-8')) resp.headers["Content-Disposition"] = "attachment; filename=realty_export.csv" resp.headers["Content-Type"] = "text/csv; charset=utf-8" return resp # === Запуск сервера === if __name__ == "__main__": init_db() # Создаёт realty.db, если нет debug_mode = os.getenv("FLASK_DEBUG", "false").lower() == "true" port = int(os.getenv("ADMIN_PORT", 5001)) host = os.getenv("ADMIN_HOST", "127.0.0.1") print(f"✅ База данных: {DATABASE_PATH}") print(f"🚀 Админка запущена: http://{host}:{port}") app.run(host=host, port=port, debug=debug_mode)