python 爬虫练手,爬取 下载 斗图啦网站前100页最新图片。、
其中知识点启用线程安全及多线程的方式进行同时下载。目前跑代码未出现异常报错。
以下实例代码
# -*- encoding=utf8 -*-
import requests
from lxml import etree
import os
import re
import threading
from queue import Queue
class P(threading.Thread):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
}
def __init__(self,page_queue,img_queue,*args,**kwargs):
super(P,self).__init__(*args,**kwargs) ##继承类的方法
self.page_queue = page_queue
self.img_queue = img_queue
def run(self):
while True:
if self.page_queue.empty():break
url = self.page_queue.get()
self.parse_page(url)
def parse_page(self,url):
text = requests.get(url, headers=self.headers).text
texthtml = etree.HTML(text)
imgs = texthtml.xpath("//div[@class='page-content text-center']//a[@class='col-xs-6 col-sm-3']//img")
for img in imgs:
imgurl = img.get('data-original')
imgname = img.get('alt')
reimg = re.sub(r'[、?()@⊙\\{}?,:></【】》《!|+.,;*]', '', imgname) #使用正则表达式过滤掉图名的特殊符号
hz = os.path.splitext(imgurl)[1]
filename = reimg + hz
self.img_queue.put((filename,imgurl))
class C(threading.Thread):
def __init__(self,page_queue,img_queue,*args,**kwargs):
super(C,self).__init__(*args,**kwargs) ##继承类的方法
self.page_queue = page_queue
self.img_queue = img_queue
def run(self):
while True:
if self.page_queue.empty() and self.img_queue.empty():
print("已下载完成")
break
filename,imgurl = self.img_queue.get()
betyimg = requests.get(imgurl,P.headers)
with open('D:\\test\\'+filename,'wb') as image: #使用二进制的方法进行保存图片
image.write(betyimg.content)
print(filename+'下载完成')
def main():
page_queue = Queue(100) #开启线程安全
img_queue = Queue(500)
##只下载前面100页
for x in range(1,101):
url = "https://www.doutula.com/photo/list/?page=%d"%x
page_queue.put(url) ##将获取到的链接保存到page_queue
for x in range(5):
t=P(page_queue,img_queue)
t.start()
for x in range(5):
t = C(page_queue,img_queue)
t.start()
if __name__ == '__main__':
main()