多线程爬虫

爬取美桌网站迪丽热巴的照片

首先进行网页解析：

http://www.win4000.com/mt/dilireba_1.html

多线程

import threading
from lxml import etree
from collections import deque
from pybloom_live import BloomFilter
from urllib import request
import time
 
 
class imgInfo:
    
    def __init__(self,url,title):
        self.url=url
        self.title=title
 
class clawer:
    
    def __init__(self,image_file_name):
        self.image_file_name=image_file_name
    
    request_header={
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
        'Connection': 'close'
        }     
    bfdownload=BloomFilter(1024*1024,0.01)
    cur_que=deque()
    
    def getImageUrl(self,url):
        req=request.Request(url,headers=self.request_header)
        response=request.urlopen(req)
        html_page=response.read()
        html=etree.HTML(html_page.lower().decode('utf-8'))
        img_list=html.xpath('//img[@title]')
        for img in img_list:
            if '迪丽热巴' in img.attrib['title'] and img.attrib['src'] not in self.bfdownload:
                self.cur_que.append(imgInfo(img.attrib['src'],img.attrib['title']))
                self.bfdownload.add(img.attrib['src'])        
        
    def getImge(self,url,title):
        req=request.Request(url,headers=self.request_header)
        response=request.urlopen(req)
        html_page=response.read()
        file=open(self.image_file_name+title+'.jpg','wb')
        file.write(html_page)
        file.close()
    
    
if __name__=='__main__':
    clw=clawer('C:\\Users\\Administrator\\Desktop\\图片\\')
    clw.getImageUrl('http://www.win4000.com/mt/dilireba_2.html')
    thread_pool=[]
    max_thread=10
    start=time.time()
    while True:
        try:           
            #多线程
            for t in thread_pool:
                 if not t.is_alive():
                     thread_pool.remove(t)
            if len(thread_pool)==max_thread:
                    continue   
            imgif=clw.cur_que.popleft()
            if imgif!=None:  
                thread=threading.Thread(target=clawer.getImge,name=None,args=(clw,imgif.url,imgif.title))
                thread_pool.append(thread)
                thread.setDaemon(True)
                thread.start()             
            else:
                break
            '''
            #单进程
            imgif=clw.cur_que.popleft()
            if imgif!=None: 
                clw.getImge(imgif.url,imgif.title)
            else:
                break
            '''
        except Exception as Arg:
            print(Arg)
            break
    print(time.time()-start)

参考：https://blog.csdn.net/dhaiuda/article/details/80980252#commentsedit