import requests
from bs4 import BeautifulSoup
headers={
'UserAgent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
}
def get_details(url):
res=requests.get(url,headers=headers)
soup=BeautifulSoup(res.text,'html.parser')
titles=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.prop-title > a')
adresses=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.info-table > div > span.info-col.row2-text > a.laisuzhou > span')
prices=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.info-table > div > div > span.total-price.strong-num')
unit_prices=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.info-table > div > span.info-col.price-item.minor')
dd=[]
for title,adress,price,unit_price in zip(titles,adresses,prices,unit_prices):
data={'title':title.get_text().strip(),'adress':adress.get_text().strip(),'price':price.get_text().strip(),'unit_price':unit_price.get_text().strip()}
dd.append(data)
return dd
url1='http://sh.lianjia.com/ershoufang/d2'
#print(get_details(url1))
'''
for i in range(1,4):
urls='http://sh.lianjia.com/ershoufang/d{}'
urlss=urls.format(i)
datas=get_details(urlss)
#print(datas)
'''
lj=[]
for i in range(1,30):
urls='http://sh.lianjia.com/ershoufang/' +'d'+str(i)
#urlss=urls.format(i)
datas=get_details(urls)
lj.append(datas)
#print(datas)
import pandas
"""
for p in range(len(lj)):
lj[p]=pandas.DataFrame(lj[p])
"""
lj=list(map(pandas.DataFrame,lj))
re= pandas.concat(lj)
re.index=list(range(re.shape[0]))
re.to_excel('re.xls')
#a1=list(map(pandas.DataFrame,lj))
链家网爬虫
最新推荐文章于 2024-08-13 21:38:27 发布