爬虫练习--豆瓣英美剧爬虫

用chorme（切换为安卓手机）到达豆瓣英美剧页面

搜索item得到json文件位置和requests请求url

爬虫中设置自己的header（要有referer，不然爬不到）并且删除多余的url

self.url_temp = "https://m.douban.com/rexxar/api/v2/subject_collection/tv_american/items?os=android&for_mobile=1&start={}&count=18&loc_id=108288&_"
self.headers = {"User-Agent": "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Mobile Safari/537.36",
                        "Referer":"https://m.douban.com/tv/american"}

代码

import requests
import json

class DoubanSpider:
    def __init__(self):
        self.url_temp = "https://m.douban.com/rexxar/api/v2/subject_collection/tv_american/items?os=android&for_mobile=1&start={}&count=18&loc_id=108288&_"
        self.headers = {"User-Agent": "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Mobile Safari/537.36",
                        "Referer":"https://m.douban.com/tv/american"}

    def pares_url(self,url): #发送请求
        print(url)
        response = requests.get(url,headers=self.headers)
        return response.content.decode()


    def get_content_list(self,json_str): #提取数据
        dict_ret = json.loads(json_str)
        #print(dict_ret)
        content_list = dict_ret["subject_collection_items"]
        return content_list


    def save_content_list(self,content_list):#保存
        with open("douban.txt","a",encoding="utf-8") as f:
            for content in content_list:
                f.write(json.dumps(content,ensure_ascii=False))
                f.write("\n")#写入换行符
        print('保存成功')


    def run(self):#实现主要逻辑输出
        num = 0
        while True:
            #1.start_url
            url = self.url_temp.format(num)
            #2.发送请求,获取响应
            json_str = self.pares_url(url)
            #3.提取数据
            content_list = self.get_content_list(json_str)
            #4.保存
            self.save_content_list(content_list)
            if len(content_list)<18:
                break
            #5.构造下一页地址,循环
            num += 18


if __name__ == "__main__":
    doubanspider = DoubanSpider()
    doubanspider.run()