/
rnekrasov
/
Python
Обзор
Документация
Войти
/
rnekrasov
/
Python
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
news_articles__scraper.py
238 строк
6 KB
slowy07
fixing except block directly handling baseException
17 июл 2021, 17:46
17 июл 2021, 17:46
b97e0a6
Код
Авторство
О чём код?
# -*- coding: utf-8 -*- """News_Articles__Scraper.ipynb Automatically generated by Colaboratory. Original file is located at https://colab.research.google.com/drive/1v1XaNvdBmHIG79KQyaVUl793rKsV7qTD ***Uncomment the line to install newspaper3k first*** """ # ! pip install newspaper3k import pickle import re import sys import urllib import pandas as pd import requests # importing necessary libraries from bs4 import BeautifulSoup from newspaper import Article # Extracting links for all the pages (1 to 158) of boomlive fake news section fakearticle_links = [] for i in range(1, 159): url = "https://www.boomlive.in/fake-news/" + str(i) try: # this might throw an exception if something goes wrong. page = requests.get(url) # send requests page = requests.get(url) soup = BeautifulSoup(page.text, "html.parser") # Collecting all the links in a list for content in soup.find_all("h2", attrs={"class": "entry-title"}): link = content.find("a") fakearticle_links.append(link.get("href")) # this describes what to do if an exception is thrown except Exception as e: # get the exception information error_type, error_obj, error_info = sys.exc_info() # print the link that cause the problem print("ERROR FOR LINK:", url) # print error info and line that threw the exception print(error_type, "Line:", error_info.tb_lineno) continue fakearticle_links[:5] len(fakearticle_links) fakearticle_links[1888:] import matplotlib.pyplot as plt import pandas as pd import numpy as np """We have to modify the links so that the links actually work as we can see that the string extracted is the last part of the url! **We have to add 'https://www.boomlive.in/fake-news' to the extracted links.** """ # Modify the links so that it takes us to the particular website str1 = "https://www.boomlive.in/fake-news" fakearticle_links = [str1 + lnk for lnk in fakearticle_links] fakearticle_links[6:10] """**The links are modified and is working :)** ***Creating a dataset of all the fake articles*** """ # Create a dataset for storing the news articles news_dataset = pd.DataFrame(fakearticle_links, columns=["URL"]) news_dataset.head() title, text, summary, keywords, published_on, author = ( [], [], [], [], [], [], ) # Creating empty lists to store the data for Url in fakearticle_links: article = Article(Url) # Call the download and parse methods to download information try: article.download() article.parse() article.nlp() except Exception as error: print(f"exception : {error}") pass # Scrape the contents of article title.append(article.title) # extracts the title of the article text.append(article.text) # extracts the whole text of article summary.append(article.summary) # gives us a summary abou the article keywords.append(", ".join(article.keywords)) # the main keywords used in it published_on.append(article.publish_date) # the date on which it was published author.append(article.authors) # the authors of the article """**Checking the lists created**""" text[6] keywords[1] published_on[6] author[6] # Adding the columns in the fake news dataset news_dataset["title"] = title news_dataset["text"] = text news_dataset["keywords"] = keywords news_dataset["published date"] = published_on news_dataset["author"] = author # Check the first five columns of dataset created news_dataset.head() """**Converting the dataset to a csv file**""" news_dataset.to_csv("Fake_news.csv") """**Reading the csv file**""" df = pd.read_csv("Fake_news.csv") # Checking the last 5 rows of the csv file df.tail(5) """**Download the csv file in local machine**""" from google.colab import files files.download("Fake_news.csv") """**Scraping news from Times of India**""" TOIarticle_links = ( [] ) # Creating an empty list of all the urls of news from Times of India site # Extracting links for all the pages (2 to 125) of boomlive fake news section for i in range(2, 126): url = "https://timesofindia.indiatimes.com/news/" + str(i) try: # send requests page = requests.get(url) soup = BeautifulSoup(page.text, "html.parser") # Collecting all the links in a list for content in soup.find_all("span", attrs={"class": "w_tle"}): link = content.find("a") TOIarticle_links.append(link.get("href")) # this describes what to do if an exception is thrown except Exception as e: # get the exception information error_type, error_obj, error_info = sys.exc_info() # print the link that cause the problem print("ERROR FOR LINK:", url) # print error info and line that threw the exception print(error_type, "Line:", error_info.tb_lineno) continue TOIarticle_links[6:15] len(TOIarticle_links) str2 = "https://timesofindia.indiatimes.com" TOIarticle_links = [str2 + lnk for lnk in TOIarticle_links if lnk[0] == "/"] TOIarticle_links[5:8] len(TOIarticle_links) title, text, summary, keywords, published_on, author = ( [], [], [], [], [], [], ) # Creating empty lists to store the data for Url in TOIarticle_links: article = Article(Url) # Call the download and parse methods to download information try: article.download() article.parse() article.nlp() except Exception: pass # Scrape the contents of article title.append(article.title) # extracts the title of the article text.append(article.text) # extracts the whole text of article summary.append(article.summary) # gives us a summary abou the article keywords.append(", ".join(article.keywords)) # the main keywords used in it published_on.append(article.publish_date) # the date on which it was published author.append(article.authors) # the authors of the article title[5] TOI_dataset = pd.DataFrame(TOIarticle_links, columns=["URL"]) # Adding the columns in the TOI news dataset TOI_dataset["title"] = title TOI_dataset["text"] = text TOI_dataset["keywords"] = keywords TOI_dataset["published date"] = published_on TOI_dataset["author"] = author TOI_dataset.head() TOI_dataset.to_csv("TOI_news_dataset.csv") dt = pd.read_csv("TOI_news_dataset.csv") dt.tail(3) from google.colab import files files.download("TOI_news_dataset.csv")