/
rnekrasov
/
Python
Обзор
Документация
Войти
/
rnekrasov
/
Python
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
twitter_post_scraper.py
39 строк
1 KB
slowy07
refactor: clean code
30 янв 2022, 04:33
30 янв 2022, 04:33
f0af0c4
Код
Авторство
О чём код?
import requests from bs4 import BeautifulSoup import re re_text = r"\:|\.|\!|(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b|(.twitter.com\/)\w*|\&" re_text_1 = r"(pictwittercom)\/\w*" def tweeter_scrapper(): list_of_dirty_tweets = [] clear_list_of_tweets = [] base_tweeter_url = "https://twitter.com/{}" tweeter_id = input() response = requests.get(base_tweeter_url.format(tweeter_id)) soup = BeautifulSoup(response.content, "lxml") all_tweets = soup.find_all("div", {"class": "tweet"}) for tweet in all_tweets: content = tweet.find("div", {"class": "content"}) message = ( content.find("div", {"class": "js-tweet-text-container"}) .text.replace("\n", " ") .strip() ) list_of_dirty_tweets.append(message) for dirty_tweet in list_of_dirty_tweets: dirty_tweet = re.sub(re_text, "", dirty_tweet, flags=re.MULTILINE) dirty_tweet = re.sub(re_text_1, "", dirty_tweet, flags=re.MULTILINE) dirty_tweet = dirty_tweet.replace(u"\xa0…", u"") dirty_tweet = dirty_tweet.replace(u"\xa0", u"") dirty_tweet = dirty_tweet.replace(u"\u200c", u"") clear_list_of_tweets.append(dirty_tweet) print(clear_list_of_tweets) if __name__ == "__main__": tweeter_scrapper()