头条抓取

import requests
from lxml import etree
import json
import time


class Toutiao(object):
    def __init__(self):
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
        }

    def get_key_word(self):
        with open('./key_word.txt', 'r') as f:
            data = f.readlines()
        return data

    # def get_url(self, data):
    #     for da in data:
    #         url_list = 'https://www.toutiao.com/api/pc/feed/?category={}&utm_source=toutiao''&widen=1&max_behot_time=0&max_behot_time_tmp=0''&tadrequire=true&as=A1256D1346DF1B7&cp=5D360F210BD7CE1''&_signature=NQO3JgAAaDs8T80zj26hTjUDtz'.format(
    #             da)
    #         for url in url_list:
    #             with open('./url.txt', 'a') as f:
    #                 f.write(url)

    def get_content(self):
        with open('./url.txt', 'r') as f:
            url_list = f.readlines()
            for url in url_list:
                try:
                    response = requests.get(url=url, headers=self.headers)
                    time.sleep(1)
                    response = json.loads(response.content.decode('utf-8'))

                    for re in response["data"]:
                        with open('../bt/bt.txt', 'a+', encoding='utf-8') as f:
                            f.write(re['title'] + '\n')

                        with open('../bt/nr.txt', 'a+', encoding='utf-8') as c:
                            c.write(re['abstract'] + '\n')
                except Exception as e:
                    print(e)


if __name__ == '__main__':
    toutiao = Toutiao()
    data = toutiao.get_key_word()
    # toutiao.get_url(data)
    toutiao.get_content()

转载于:https://www.cnblogs.com/victorstudy/p/11237288.html