Python bs4抓取新浪新闻简单版

最新推荐文章于 2022-11-26 23:41:14 发布

wwxy261

最新推荐文章于 2022-11-26 23:41:14 发布

阅读量341

点赞数

分类专栏：爬虫

本文链接：https://blog.csdn.net/wwxy1995/article/details/80934678

版权

爬虫专栏收录该内容

5 篇文章 0 订阅

订阅专栏

import requests
from bs4 import BeautifulSoup
import time
import pandas

res = requests.get("http://news.sina.com.cn/china/")
res.encoding = "utf-8"
soup = BeautifulSoup(res.text, 'html.parser')
newsary = []


def get_article(url):
    res = requests.get(url)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    dic = {}
    if len(soup.select(".main-title"))> 0 :
        dic['title'] = soup.select(".main-title")[0].text
    if len(soup.select('.article'))>0:
        dic['content'] = ''.join(soup.select('.article')[0].text.split())
    if len(soup.select('.keywords'))>0:
        dic['keywords'] = soup.select('.keywords')[0].text
    if len(soup.select('.date-source'))>0:
        dic['sourse'] = soup.select('.date-source')[0].text
    return dic

count = 0
for link in soup.select('.news-item'):
    if len(link.select('h2 a'))>0:
        newsary.append(get_article(link.select('h2 a')[0]['href']))
        count +=1
        print("已获取第" + str(count) + "条新闻")
    # time.sleep(1)
df = pandas.DataFrame(newsary)
df.to_excel("xinlang_news.xlsx")