批量爬取多家公司新闻

1 需求

批量爬取多家公司新闻，并实现持续化存储。

2 代码实现

import requests
import re


def baidu(company):
    # 获取网页源代码
    url = 'https://www.baidu.com/s?rtt=4&tn=news&word=' + company
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'}
    source = requests.get(url=url, headers=headers).text
    # 获取网页标题
    p_title = '<h3 class="news-title_1YtI1">.*?<!--s-text-->(.*?)<!--/s-text--></a></h3>'
    title = re.findall(p_title, source, re.S)
    # 获取网页网址
    p_href = '<h3 class="news-title_1YtI1"><a href="(.*?)" target="_blank"'
    href = re.findall(p_href, source)
    # 数据清洗
    filel = open('./数据挖掘报告.txt', 'a')
    filel.write(company + "数据挖掘完毕！" + '\n' + '\n')
    for index in range(len(title)):
        title[index] = re.sub('<.*?>', '', title[index]).strip()
        filel.write(str(index + 1) + "." + title[index] + "(" + href[index] + ")" + "\n")
        filel.write("————————————————————" + "\n" + "\n")

companies = ['华能信托', '百度集团', '阿里巴巴']
for index in companies:
    baidu(index)
    print(index + '百度新闻爬器成功！')