今天早上看着了人家用 js解密方法去爬漫画,心潮澎湃试了一下,,,,人家漫画网站改版了,,,
在用框架试一下吧,,,额,,看不懂报错我搞啥啊
所以贼菜的我还是老老实实用多线程吧,,,,
然后找了半天能爬的网站,最终还是漫客栈吧,,,(人家都有js加密,,这个没有)
import requests,threading,os,time
from queue import Queue
from lxml import etree
from fake_useragent import UserAgent
ua = UserAgent()
headers = {'user-agent': ua.random}
url = 'https://www.mkzhan.com/210839/' # 这个是龙王传说的地址,可以换上你喜欢的
class Producer(threading.Thread):
def __init__(self,url_queue):
super(Producer, self).__init__()
self.url_queue=url_queue
def run(self) -> None:
resp = requests.get(url,headers=headers)
html = etree.HTML(resp.text)
detail_urls = html.xpath('//ul[@class="chapter__list-box clearfix hide"]//li/a/@data-hreflink')
for detail_url in detail_urls:
detail_url = 'https://www.mkzhan.com'+detail_url
self.url_queue.put(detail_url)
print(self.url_queue.qsize())
class Consumer(threading.Thread):
def __init__(self,url_queue):
super(Consumer, self).__init__()
self.url_queue = url_queue
def run(self) -> None:
while not self.url_queue.empty():
count = 0
url = self.url_queue.get()
resp = requests.get(url,headers=headers)
html = etree.HTML(resp.text)
image_urls = html.xpath(r'//div[@class="rd-article__pic hide"]//@data-src')
title = html.xpath(r'/html/body/div[2]/div[1]/div[2]/h1/a/text()')
save_path =os.path.join(r'C:\Users\NERO\Desktop\漫画',title[0])
if not os.path.exists(save_path):
os.mkdir(save_path)
for image_url in image_urls:
path = os.path.join(save_path, (str(count) + '.jpg'))
count += 1
with open(path, 'wb') as f:
resp = requests.get(image_url)
f.write(resp.content)
print('%s下载完成'%title[0])
def main():
url_queue = Queue(400)
th = Producer(url_queue)
th.start()
time.sleep(3)
for x in range(5):
th = Consumer(url_queue)
th.start()
if __name__ == '__main__':
main()
额,可能是文件的读取操作很耗时间吧,,urlretrieve 不好使啊,有格式问题,,总之就是爬的很慢,,,以后再改进吧