简单小爬虫之内涵段子网

#内涵段子的小爬虫

# coding=utf-8
import requests
from retrying import retry
import re
import json


class NeihanSpider(object):
    def __init__(self):
        self.start_url = "https://neihanshequ.com/"
        self.headers = {
            "User-Agent": "Mozilla/5.0(Macintosh; Intel Mac OS X 10_12_4) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 60.0.3112.113Safari / 537.36"
        }
        self.next_url_temp = "https://neihanshequ.com/joke/?is_json=1&app_name=neihanshequ_web&max_time={}"

    @retry(stop_max_attempt_number=3)
    def _parse_url(self, url):
        response = requests.get(url, headers=self.headers, timeout=3)
        assert response.status_code == 200
        return response.content.decode()

    def parse_url(self, url):
        print("now parseing", url)
        try:
            return self._parse_url(url)
        except Exception as e:
            print(e)
            return None

    def get_content_list(self, html_str):
        content_list = re.findall(r"<h1 class=\"title\">.*?<p>(.*?)</p>", html_str, re.S)
        max_time = re.findall(r"max_time: '(.*?)',", html_str)

        return content_list, max_time[0]

    def save_content_list(self, content_list):
        file_path = "./duanzi.txt"
        with open(file_path, "a", encoding='utf-8')as f:
            for content in content_list:
                f.write(content)
                f.write("\n")

    def get_next_page_content(self, url):
        json_response = self.parse_url(url)
        dict_response = json.loads(json_response)
        content_list = dict_response["data"]["data"]
        content_list = [i["group"]["text"] for i in content_list]
        max_time = dict_response["data"]["max_time"]
        has_more = dict_response["data"]["has_more"]
        return content_list, max_time, has_more

    def run(self):
        # 1.start_url
        # 2.发送请求，获取相应
        html_str = self.parse_url(self.start_url)
        # 3.提取数据
        if html_str is not None:
            content_list, max_time = self.get_content_list(html_str)

            # 4.保存
            self.save_content_list(content_list)
            # 5.获取下一页的内容
            has_more = True
            while has_more:
                next_url = self.next_url_temp.format(max_time)
                content_list, max_time, has_more = self.get_next_page_content(next_url)
                self.save_content_list(content_list)


if __name__ == '__main__':
    neihan = NeihanSpider()
    neihan.run()