爬取美桌网站迪丽热巴的照片
首先进行网页解析:
http://www.win4000.com/mt/dilireba_1.html
多线程
import threading
from lxml import etree
from collections import deque
from pybloom_live import BloomFilter
from urllib import request
import time
class imgInfo:
def __init__(self,url,title):
self.url=url
self.title=title
class clawer:
def __init__(self,image_file_name):
self.image_file_name=image_file_name
request_header={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'Connection': 'close'
}
bfdownload=BloomFilter(1024*1024,0.01)
cur_que=deque()
def getImageUrl(self,url):
req=request.Request(url,headers=self.request_header)
response=request.urlopen(req)
html_page=response.read()
html=etree.HTML(html_page.lower().decode('utf-8'))
img_list=html.xpath('//img[@title]')
for img in img_list:
if '迪丽热巴' in img.attrib['title'] and img.attrib['src'] not in self.bfdownload:
self.cur_que.append(imgInfo(img.attrib['src'],img.attrib['title']))
self.bfdownload.add(img.attrib['src'])
def getImge(self,url,title):
req=request.Request(url,headers=self.request_header)
response=request.urlopen(req)
html_page=response.read()
file=open(self.image_file_name+title+'.jpg','wb')
file.write(html_page)
file.close()
if __name__=='__main__':
clw=clawer('C:\\Users\\Administrator\\Desktop\\图片\\')
clw.getImageUrl('http://www.win4000.com/mt/dilireba_2.html')
thread_pool=[]
max_thread=10
start=time.time()
while True:
try:
#多线程
for t in thread_pool:
if not t.is_alive():
thread_pool.remove(t)
if len(thread_pool)==max_thread:
continue
imgif=clw.cur_que.popleft()
if imgif!=None:
thread=threading.Thread(target=clawer.getImge,name=None,args=(clw,imgif.url,imgif.title))
thread_pool.append(thread)
thread.setDaemon(True)
thread.start()
else:
break
'''
#单进程
imgif=clw.cur_que.popleft()
if imgif!=None:
clw.getImge(imgif.url,imgif.title)
else:
break
'''
except Exception as Arg:
print(Arg)
break
print(time.time()-start)
参考:https://blog.csdn.net/dhaiuda/article/details/80980252#commentsedit