python爬虫之提取糗事百科前5页的内容

'''
糗事百科前5页内容爬取
类的使用
'''
# 导入第三方库
import requests
from lxml import etree
import re


# 定义糗事百科的类
class QiuShiBaiKe():
    # 初始化对象
    def __init__(self):
        self.start_url = "http://www.qiushidabaike.com/index_{}.html"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36"}

    # 定义得到url列表的方法
    def get_url_list(self):
        return [self.start_url.format(i) for i in range(1, 5)]

    # 定义解析html的方法
    def paser_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    # 定义得到要提取文本的方法
    def get_content_list(self, html_str):
        html = etree.HTML(html_str)
        dl_list = html.xpath('//dl[@class="main-list"]')
        content = []
        for dl in dl_list:
            item = {}
            item["content"] = dl.xpath('./dd[@class="content"]/p/text()')
            item["content"] = [item["content"][0] if len(item["content"]) > 0 else None]
            item["auther"] = dl.xpath('.//p[@class="user"]/a/text()')
            item["publish_time"] = dl.xpath('.//p[@class="user"]/span[@class="fr"]/text()')
            # 将文本中的\t|\r|\n|\xa0替换为空
            item["publish_time"] = [re.sub('\t|\r|\n|\xa0', '', i) for i in item["publish_time"]]
            content.append(item)
        return content

    # 定义运行函数
    def run(self):
        url_list = self.get_url_list()
        for url in url_list:
            html_str = self.paser_url(url)
            self.get_content_list(html_str)


# 程序运行接口
if __name__ == '__main__':
    QiuShi_Spider = QiuShiBaiKe()
    QiuShi_Spider.run()