# -*- coding: utf-8 -*-
"""
Created on Wed Feb 7 23:33:26 2018
@author: Administrator
"""
'''
农产品集购网获取现货商城信息
'''
import requests
from bs4 import BeautifulSoup
import time
from multiprocessing import Pool
headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'}
#url='http://www.16988.com/fronts/spotmall/0/0/0/0/0/0/0/3/0?searchKeyword=&searchBrand=&is_odd=&searchCompany='
total=[]
def get_nongchanpin(url):
res=requests.get(url,headers=headers)
print(res.status_code)
soup=BeautifulSoup(res.text,'html.parser')
all_chanpin=soup.find_all('ul',class_='foreach-table-tr d-hover-border ')
for each in all_chanpin:
pinming=each.find('li',class_='productname').text
guige=each.find('li',class_='pecifications').text
pinpai=each.find('li',class_='brands').text
cangku=each.find('li',class_='warehouse').text
try:
qidingliang=int(each.find('li',class_='orderquantity').text)
except ValueError:
qidingliang=float(each.find('li',class_='orderquantity').text)
try:
chicangliang=int(each.find('li',class_='positions').text)
except ValueError:
chicangliang=float(each.find('li',class_='positions').text)
zoushi=each.find('li',class_='foreach-pricetrend pricetrend').font.text
zoushi_jiantou=each.find('li',class_='foreach-pricetrend pricetrend').img['src']
jiage=each.find('li',class_='foreach-price price').text.strip()
info={'品名':pinming,
'规格':guige,
'品牌':pinpai,
'仓库':cangku,
'起订量(吨)':qidingliang,
'持仓量(吨)':chicangliang,
'走势':zoushi,
'箭头':zoushi_jiantou,
'价格(元/吨)':jiage}
total.append(info)
return total
if __name__ == '__main__':
for i in range(501,967):
url='http://www.16988.com/fronts/spotmall/0/0/0/0/0/0/0/'+str(i)+'/0?searchKeyword=&searchBrand=&is_odd=&searchCompany='
get_nongchanpin(url)
time.sleep(0.2)
'''
if __name__ == '__main__':
urls=['http://www.16988.com/fronts/spotmall/0/0/0/0/0/0/0/{}/0?searchKeyword=&searchBrand=&is_odd=&searchCompany='.format(i) for i in range(0,5)]
start=time.time()
pool=Pool()
pool.map(get_nongchanpin,urls)
end=time.time()
print('时间:',end-start)
'''
import pandas as pd
df=pd.DataFrame(total)
df.to_excel('nongzuowu2.xls',columns=['品名','规格','品牌','仓库','起订量(吨)','持仓量(吨)','走势','箭头','价格(元/吨)'])
python农产品集购网获取现货商城信息
最新推荐文章于 2024-05-31 21:43:53 发布