爬虫案例：漫客栈漫画下载

今天早上看着了人家用 js解密方法去爬漫画，心潮澎湃试了一下，，，，人家漫画网站改版了，，，

在用框架试一下吧，，，额，，看不懂报错我搞啥啊

所以贼菜的我还是老老实实用多线程吧，，，，

然后找了半天能爬的网站，最终还是漫客栈吧，，，（人家都有js加密，，这个没有）

import requests,threading,os,time
from queue import Queue
from lxml import etree
from fake_useragent import UserAgent


ua = UserAgent()
headers = {'user-agent': ua.random}
url = 'https://www.mkzhan.com/210839/' # 这个是龙王传说的地址，可以换上你喜欢的

class Producer(threading.Thread):
    def __init__(self,url_queue):
        super(Producer, self).__init__()
        self.url_queue=url_queue

    def run(self) -> None:
        resp = requests.get(url,headers=headers)
        html = etree.HTML(resp.text)
        detail_urls = html.xpath('//ul[@class="chapter__list-box clearfix hide"]//li/a/@data-hreflink')
        for detail_url in detail_urls:
            detail_url = 'https://www.mkzhan.com'+detail_url
            self.url_queue.put(detail_url)
        print(self.url_queue.qsize())


class Consumer(threading.Thread):
    def __init__(self,url_queue):
        super(Consumer, self).__init__()
        self.url_queue = url_queue

    def run(self) -> None:
        while not self.url_queue.empty():
            count = 0
            url = self.url_queue.get()
            resp = requests.get(url,headers=headers)
            html = etree.HTML(resp.text)
            image_urls = html.xpath(r'//div[@class="rd-article__pic hide"]//@data-src')
            title = html.xpath(r'/html/body/div[2]/div[1]/div[2]/h1/a/text()')
            save_path =os.path.join(r'C:\Users\NERO\Desktop\漫画',title[0])
            if not os.path.exists(save_path):
                os.mkdir(save_path)
            for image_url in image_urls:
                path = os.path.join(save_path, (str(count) + '.jpg'))
                count += 1
                with open(path, 'wb') as f:
                    resp = requests.get(image_url)
                    f.write(resp.content)
            print('%s下载完成'%title[0])

def main():
    url_queue = Queue(400)
    th = Producer(url_queue)
    th.start()
    time.sleep(3)
    for x in range(5):
        th = Consumer(url_queue)
        th.start()

if __name__ == '__main__':
    main()

额，可能是文件的读取操作很耗时间吧，，urlretrieve 不好使啊，有格式问题，，总之就是爬的很慢，，，以后再改进吧