python爬虫——使用代理和xpath爬取豆瓣读书

根据豆瓣读书的所有标签自动创建文件夹，使用代理防止被反爬。但是我的代理好像是假的，还是被反爬了…通过设置cookie爬取，但是爬取一定数量后需要在浏览器手动进行验证码输入…总的来说，代码写的很麻烦

import requests
from lxml import etree
from fake_useragent import UserAgent
import threading
import queue
import os
from urllib.parse import urljoin
import json
import csv
# 导入代理池
from proxies.get_proxy import get_ip_port
import random


# 获取ip
temp = list(map(lambda x: x[1], get_ip_port()))
temp = [x for x in temp if x]
# 构建请求头
ua = UserAgent()
headers = {
    'user-agent': ua.Chrome,
    'Cookie': '自己设置'
    }
# 创建锁
LOCK = threading.Lock()

# 创建全局队列
q_url = queue.Queue()
q_path = queue.Queue()


# 创建多线程
class MyThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        # 声明一个列表存储字典
        self.data_list = []
        LOCK.acquire()
        self.path = q_path.get()
        self.url = q_url.get()
        self.flag = False
        LOCK.release()

    def run(self) -> None:
        self.parse_url(self.url, random.choice(temp))
        self.save_file()

    # 解析url，爬取数据
    def parse_url(self, temp_url, proxy):
        if not self.flag:
            # 设置代理
            proxies = {
                'https:': random.choice(temp)
            }
        else:
            proxies = proxy

        try:
            url = temp_url
            resp = requests.get(url, headers=headers, proxies=proxies, timeout=3)
            if resp.status_code == 200:

                self.flag = True

                html = etree.HTML(resp.content)
                lis = html.xpath('//ul[@class="subject-list"]/li')
                if lis:
                    for li in lis:
                        # 声明一个字典存储数据
                        data_dict = {}
                        # 题目
                        title = li.xpath('.//h2/a/text()')[0]
                        # 清洗数据
                        title = title.strip()
                        data_dict['title'] = title
                        # 作者、出版社、年份、价格
                        all_info = li.xpath('.//div[@class="pub"]/text()')[0]
                        # 清洗数据
                        all_info = all_info.strip().split('/')
                        # 判断作者有多少个，如果all_info的长度是4那么作者有一个，如果是5个那么作者是2个
                        if len(all_info) == 4:
                            # 作者
                            author = all_info[0]
                            # 出版社
                            pub_house = all_info[1]
                            # 日期
                            datetime = all_info[2]
                            price = all_info[3]

                            data_dict['author'] = author
                            data_dict['pub_house'] = pub_house.strip()
                            data_dict['datetime'] = datetime.strip()
                            data_dict['price'] = price.strip()
                        elif len(all_info) == 5:
                            # 作者
                            author = all_info[0] + '/' + all_info[1]
                            # 出版社
                            pub_house = all_info[2]
                            # 日期
                            datetime = all_info[3]
                            price = all_info[4]

                            data_dict['author'] = author
                            data_dict['pub_house'] = pub_house.strip()
                            data_dict['datetime'] = datetime.strip()
                            data_dict['price'] = price.strip()
                        # 评分
                        rating_nums = li.xpath('.//span[@class="rating_nums"]/text()')
                        if rating_nums:
                            rating_nums = rating_nums[0].strip()
                        # 评论数
                        comment_count = li.xpath('.//span[@class="pl"]/text()')[0]
                        # 清洗数据
                        comment_count = comment_count.strip().replace('(', '').replace('人评价)', '')
                        # 摘要
                        text = li.xpath('.//p/text()')
                        if text:
                            # 数据清洗
                            text = text[0].strip().replace('\r', '').replace('\n', '')
                        else:
                            text = None

                        data_dict['rating_nums'] = rating_nums
                        data_dict['comment_count'] = comment_count
                        data_dict['text'] = text

                        print(data_dict)
                        self.data_list.append(data_dict)

                next_url = html.xpath('//span[@class="next"]/a/@href')[0]
                # 拼接url
                if next_url:
                    next_url = urljoin(url, next_url)
                    print(next_url)
                    return self.parse_url(next_url, proxies)
            else:
                self.flag = False

                return self.parse_url(url, proxies)

        except Exception as e:
            LOCK.acquire()
            q_url.put(self.url)
            q_path.put(self.path)
            LOCK.release()

    # 将数据存入json文件和csv文件
    def save_file(self):

        if self.data_list:
            if not os.path.exists(self.path + '.json'):
                with open(self.path + '.json', 'a+', encoding='utf-8') as f:
                    json.dump(self.data_list, f, ensure_ascii=False, indent=4)
                print('json文件写入完成')
            if not os.path.exists(self.path + '.csv'):
                with open(self.path + '.csv', 'w', encoding='utf-8', newline='') as f:
                    # 表头
                    title = self.data_list[0].keys()
                    # 创建writer对象
                    writer = csv.writer(f)
                    # 写入表头
                    writer.writerow(title)
                    # 批量写入
                    for row in self.data_list:
                        writer.writerow(row.values())
                print('csv文件写入完成')


# 获取标签名字并新建文件夹
def get_tag():
    # 文件的基本路径
    path = r'E:\PycharmCode\dbds_spiders'
    # 设置代理
    proxies = {
        'https': random.choice(temp)
    }
    # 声明大标签
    big_tag_list = []
    # 获取标签名字的链接
    tag_url = 'https://book.douban.com/tag/?view=type&icn=index-sorttags-all'
    try:
        # 请求url
        resp = requests.get(tag_url, headers=headers, proxies=proxies, timeout=3)
        if resp.status_code is 200:
            # 将返回体转换成xpath的格式
            html = etree.HTML(resp.content)
            # 获取大标签的名字
            big_tag = html.xpath('//div[@class="article"]/div[2]/div')
            # 遍历大标签
            for tag in big_tag:
                # 获取标签名，原始数据是文学 · · · · · ·
                tag_name = tag.xpath('.//h2/text()')[0]
                # 数据清洗一下
                big_tag_name = tag_name.split('·')[0].strip()
                big_tag_list.append(big_tag_name)

                # 获取小标签的名字
                # 获取每个大标签下的小标签
                tds = tag.xpath('.//td')
                # 获取每个小标签的名字
                for td in tds:
                    min_tag_name = td.xpath('./a/text()')[0]
                    min_tag_url = td.xpath('./a/@href')[0]
                    min_tag_url = urljoin(tag_url, min_tag_url)
                    # 组合路径
                    file_path = path + '\\' + big_tag_name + '\\' + min_tag_name
                    # 将路径和url放入对于队列
                    q_url.put(min_tag_url)
                    q_path.put(file_path)

            return big_tag_list
        else:
            return get_tag()
    except Exception as e:
        return get_tag()


# 创建文件
def make_file(big_tag_list):
    # 文件的基本路径
    path = r'E:\PycharmCode\dbds_spiders'

    # 判断文件是否存在，如果不存在就新建文件夹
    # 新建文件夹路径
    for big_temp in big_tag_list:
        big_path = path + '\\' + big_temp
        # 新建文件夹
        if not os.path.exists(big_path):
            os.makedirs(big_path)

    print('文件夹新建完成')


def main():

    # 先将小标签的路径和url提取出来并放入队列
    big_tag_list = get_tag()
    make_file(big_tag_list)

    while not q_url.empty() and not q_path.empty():
        # print(q_url.get())
        # print(q_path.get())
        # print('++++++++++++++++++++++++')
        ths = []
        # 每次创建3个线程
        for i in range(3):
            t = MyThread()
            ths.append(t)
        for t in ths:
            t.start()
        for t in ths:
            t.join()


if __name__ == '__main__':

    main()