网络爬虫（三）

爬取人民网新闻

导包

import requests
import re
import chardet

发送请求，获取响应

def get_html(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}
    response = requests.get(url, headers=headers)

    # 将response编码设定为与网站所用编码一致，从而使response.text不会乱码。
    response.encoding = chardet.detect(response.content)['encoding']
    if response.status_code == 200:
        return response.text      # 字符串类型

保存数据

# 保存数据
def save_data(title, date, sourse, article):
    # with open('{}.txt'.format(date[:11] + '_' + title), 'w', encoding='utf-8') as f:
    with open('%s.txt' % (date[:11] + '_' + title), 'w', encoding='utf8') as f:
        f.write(title + '\t' + date + '\t' + sourse + '\n')
        for item in article:
            f.write(item.strip() + '\n')

解析响应

def parse_html(html):
    prog = re.compile(r'<h1>(.*?)&nbsp;(.*?)</h1>')
    # title = prog.findall(html)
    title = re.findall(prog, html)
    title = title[0][0] + title[0][1]   # 字符串
    # print(title)
    prog = re.compile(r'<div class="box01">.*?>(.*?)&nbsp;', re.S)
    date = re.findall(prog, html)[0]   # 字符串
    # print(date)
    prog = re.compile(r'<div class="box01">.*?target="_blank">(.*?)</a>', re.S)
    sourse = re.findall(prog, html)[0]   # 字符串
    # print(sourse)
    prog = re.compile(r'<p style="text-align: center;">(.*?)<div class="zdfy clearfix">', re.S)
    article = re.findall(prog, html)[0]
    prog = re.compile(r'<p>(.*?)</p>', re.S)
    article = re.findall(prog, article)       # 列表
    # 保存数据
    save_data(title, date, sourse, article)

主函数

def main():
    url = 'http://tc.people.com.cn/n1/2019/0227/c183008-30906013.html'

    # 发送请求，获取响应
    html = get_html(url)
    # print(html)

    # 解析响应
    parse_html(html)

程序入口

if __name__ == '__main__':
    main()