scripts/server_parse.py - master - ndvkerch/deputatbot

scripts/server_parse.py
81 строка2 KB
WindSpot Team
Fix knowledge base: auto-inject city_id and add management scripts
21 фев 2026, 11:27
21 фев 2026, 11:27f669b27
#!/usr/bin/env python3
"""Server-side parse-only script. Runs VK/TG parsing + chunking, NO vectorization.

Intended to be executed on the server where VK tokens are valid.
Called by: manage_knowledge.py server-parse
"""
from __future__ import annotations

import asyncio
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))


async def main() -> None:
    from app.config import get_settings
    from app.db.database import Database
    from app.db.init_db import init_db
    from app.db.repositories.knowledge_repo import KnowledgeRepository
    from app.services.source_sync import SourceSyncService
    from app.services.vk_parser import VKParser
    from app.services.token_vault import TokenVault
    from app.services.vk_token_manager import VKTokenManager

    settings = get_settings()
    db = Database(settings.db_full_path)
    await db.connect()

    vault = TokenVault(settings.token_encryption_key)
    await init_db(db, settings.embedding_dimension, vault=vault)

    tm = VKTokenManager(
        db=db, vault=vault,
        app_id=settings.vk_app_id,
        app_secret=settings.vk_app_secret,
    )

    vk_parser = VKParser(
        app_id=settings.vk_app_id,
        app_secret=settings.vk_app_secret,
        api_version=settings.vk_api_version,
        db=db,
        token_manager=tm,
    )

    sync = SourceSyncService(
        db=db,
        embedding_service=None,
        vk_parser=vk_parser,
        chunk_size=settings.knowledge_chunk_size,
        chunk_overlap=settings.knowledge_chunk_overlap,
    )

    repo = KnowledgeRepository(db)
    sources = await repo.get_all_active_sources()

    if not sources:
        print("No active sources found.")
        await db.disconnect()
        return

    print(f"Parsing {len(sources)} source(s)...")
    total_posts = 0
    for src in sources:
        try:
            n = await sync.sync_single_source(src.id)
            total_posts += n
            posts_in_db = await repo.get_posts_count_by_source(src.id)
            print(f"  {src.display_name}: {n} new posts ({posts_in_db} total)")
        except Exception as e:
            print(f"  {src.display_name}: ERROR {e}")

    chunks = await sync._chunk_pending()
    print(f"\nDone: {total_posts} new posts, {chunks} chunks created")

    await db.disconnect()


if __name__ == "__main__":
    asyncio.run(main())