多协程抓取薄荷网食物热量下篇

最新推荐文章于 2021-02-10 00:02:48 发布

不懂就问问

最新推荐文章于 2021-02-10 00:02:48 发布

阅读量394

点赞数

分类专栏： python 文章标签： gevent queue requests openpyxl

本文链接：https://blog.csdn.net/net_illiteracy/article/details/101227221

版权

python 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

如题
from gevent import monkey
monkey.patch_all
import gevent,requests,openpyxl,time
from gevent.queue import Queue
from bs4 import BeautifulSoup
start=time.time()
headers={
‘User-Agent’:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36’
}
url_master=[‘http://www.boohee.com/food/view_menu’]
url_list=[]
url_dict={}
for i in range(1,11):
u_i=‘http://www.boohee.com/food/group/’+str(i)
url_master.append(u_i)

for i in url_master:
url_slave=[]
for j in range(1,11):
url_slave.append(i+’?page=’+str(j))
url_list.append(i+’?page=’+str(j))
url_dict[i]=url_slave

value_list=[]
for value in url_dict.values():
value_list.append(value)

work=Queue()
for url in url_list:
work.put_nowait(url)

content=[]
def spider():
global value_list,zero,one,two,three,four,five,six,seven,eigth,nine,ten
while not work.empty():
url=work.get_nowait()
if url in value_list[0]:
zero=res(url)
elif url in value_list[1]:
one=res(url)
elif url in value_list[2]:
two=res(url)
elif url in value_list[3]:
three=res(url)
elif url in value_list[4]:
four=res(url)
elif url in value_list[5]:
five=res(url)
elif url in value_list[6]:
six=res(url)
elif url in value_list[7]:
seven=res(url)
elif url in value_list[8]:
eigth=res(url)
elif url in value_list[9]:
nine=res(url)
elif url in value_list[10]:
ten=res(url)

def res(url):
res=requests.get(url,headers=headers)
#print(url,res.status_code)
soup=BeautifulSoup(res.text,‘html.parser’)
a=soup.find_all(‘div’,class_=‘text-box pull-left’)
for i in a:
title=i.find(‘a’)[‘title’]
href=‘http://www.boohee.com’+i.find(‘a’)[‘href’]
calorie=i.find(‘p’).text
content.append([title,href,calorie])
return content

def write(sheet,list):
for i in list:
sheet.append(i)

task_list=[]
for x in range(5):
task=gevent.spawn(spider)
task_list.append(task)
gevent.joinall(task_list)

wb=openpyxl.Workbook()
name_list=[‘菜肴’,‘谷薯芋、杂豆、主食’,‘蛋类、肉类及制品’,‘奶类及制品’,‘蔬果和菌藻’,‘坚果、大豆及制品’,‘饮料’,‘食用油、油脂及制品’,‘调味品’,‘零食、点心、冷饮’,‘其他’]
sheet=wb.active
sheet.title=name_list[0]
for i in range(1,11):
wb.create_sheet(name_list[i])
for i in name_list:
sheet=wb[i]
sheet[‘A1’]=‘菜名’
sheet[‘B1’]=‘链接’
sheet[‘C1’]=‘卡路里’

content_list=[zero,one,two,three,four,five,six,seven,eigth,nine,ten]
for i in range(11):
write(wb[name_list[i]],content_list[i])

wb.save(‘boohee.xlsx’)
wb.close()

end=time.time()
print(end-start)