load_data.py - master - JoraFreeman/DocuWiz

load_data.py
52 строки2 KB
JoraFreeman
весь нужный код
24 май 2025, 22:19
24 май 2025, 22:193974658
import asyncio
import time
import aiohttp

class AsyncDownloadUrls:
    CHUNK_SIZE = 1000
    
    @staticmethod
    def github_to_raw_url(url):
        if "github.com" in url and "/blob/" in url:
            return url.replace("github.com", "raw.githubusercontent.com").replace("/blob/", "/")
        return url

    @staticmethod
    async def async_download_single(session, url, chunk_size):
        raw_url = AsyncDownloadUrls.github_to_raw_url(url)
        print(f"[Асинхронно] Загружаю {raw_url}")
        try:
            async with session.get(raw_url, ssl=False) as response:
                if response.status != 200:
                    raise Exception(f"Ошибка: {response.status} для {raw_url}")

                content = await response.read()
                chunks = [content[i:i+chunk_size] for i in range(0, len(content), chunk_size)]
                return raw_url, chunks
        except Exception as e:
            print(f"[Ошибка] Не удалось загрузить {raw_url}: {e}")
            return raw_url, []

    @classmethod
    async def async_download_all(cls, urls):
        start_time = time.time()
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
        }
        async with aiohttp.ClientSession(headers=headers) as session:
            tasks = [cls.async_download_single(session, url, cls.CHUNK_SIZE) for url in urls]
            results = await asyncio.gather(*tasks)

        end_time = time.time()
        print(f"[Асинхронно] Всего времени: {end_time - start_time:.2f} секунд\n")
        return dict(results)

    @staticmethod
    def read_urls_from_file(filename="LANG.txt"):
        try:
            with open(filename, "r", encoding="utf-8") as f:
                urls = [line.strip() for line in f if line.strip()]
                return urls
        except FileNotFoundError:
            print(f"[Ошибка] Файл {filename} не найден.")
            return []