完整代码
import requests
import pandas as pd
from bs4 import BeautifulSoup
import time
data=pd.DataFrame()
for i in range(1,10):
print(‘正在爬取第%d页’%i)
url = ‘http://piao.qunar.com/ticket/list.htm?keyword=北京&page={}’.format(i)
headers = {‘User-Agent’:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36’}
response = requests.get(url,headers=headers)
response.encoding=‘utf-8’
soup = BeautifulSoup(response.text,‘lxml’)#对html进行解析,完成初始化
html=soup.select(‘div [class=“sight_item”]’)
itemslist=[]
for a in html:
it=[]
it.append(a[“data-sight-name”])#景点名称
it.append(a["data-address"]) #地址
b=a.select('div[class="clrfix"]')
for i in range(len(b)):
c=b[i].get_text()[0:4] #等级
if "景区" in c:
it.append(c)
else:
it.append("无等级")
attr_list = a.select('span[class="sight_item_price"]') # 门票
for i in attr_list:
it.append((i.get_text().split()[0])+(i.get_text().split()[1]))
attr_list = a.select('span[class="hot_num"]') # 销售数量
for a in attr_list:
it.append(a.get_text())
itemslist.append(it)
df=pd.DataFrame(itemslist)
data=pd.concat([data,df])
time.sleep(2)
data.columns=[‘景点名称’,‘地址’,‘等级’,‘门票’,‘销售数量’]
data.set_index(‘景点名称’)
爬取结果