/
dpavlov
/
moex-datagrid
Обзор
Документация
Войти
/
dpavlov
/
moex-datagrid
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
scripts/load_html_text.py
43 строки
1 KB
Dmitriy Pavlov
experimental crawler
19 апр 2026, 13:53
19 апр 2026, 13:53
51db5df
Код
Авторство
О чём код?
import requests from bs4 import BeautifulSoup def extract_text(url): headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/122.0.0.0 Safari/537.36" ) } try: response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() except requests.RequestException as exc: print(f"[ERROR] Не удалось загрузить текст страницы {url}: {exc}") return None soup = BeautifulSoup(response.text, "html.parser") for tag in soup(["script", "style", "header", "footer", "nav", "aside"]): tag.decompose() main = soup.find("article") if not main: main = soup.find("div", class_="content") if not main: main = soup.body if not main: return None return main.get_text(separator="\n", strip=True) if __name__ == "__main__": url = "https://bcs-express.ru/targets" txt = extract_text(url) if txt: print(txt[:50000]) with open("article2.txt", "w", encoding="utf-8") as file: file.write(txt)