python中实现多进程主要在 multiprocessing库中实现,之前利用分布式爬虫与普通爬虫做了一个对比,部分代码如下:
import multiprocessing as mp
class craw_method(analysis):
# 获取最近100条新闻信息
def norml(self,):
# result_content.colu
list_num = 10 # 第一页
tit_count = 0 # 总新闻数
while list_num < 15:
info_content = dict()
url = 'http://news.hbue.edu.cn/jyyw/list{}.htm'.format(list_num)
tit_url_bag = self.get_detailed_url_title(url) # 获取新闻题目及详细页
for unit in tit_url_bag:
sub_info, content_final = self.get_detailed_info(unit[0]) # 获取详细页信息
print(sub_info)
list_num+=1
def multiprocess(self,):
i=0
pool=mp.Pool(8)
urls=['http://news.hbue.edu.cn/jyyw/list{}.htm'.format(k)
for k in range(10,15) ]
tit_url_bag =[pool.apply_async(self.get_detailed_url_title,args=(url,)) for url in urls]
url_title=[]
for k in tit_url_bag:
url_title += k.get()
#print(url_title)
res = [pool.apply_async(self.get_detailed_info, args=(unit[0],)) for unit in url_title] # 获取详细页信息
for k in res:
print(k.get()[0])
'''
for unit in url_title:
info_content = dict()
try:
i+=1
except:
continue
print('爬取完毕,大小为%s' % i)
'''
Pool方法创建进程池,apply_async实现异步非阻塞,最终爬虫时间:
multiprocess用时:4.320017576217651
normal用时:19.634695768356323
提升4.5倍