1、爬虫思路分析
(1)原来的北京短租房的地址规则如下:
http://bj.xiaozhu.com/
http://bj.xiaozhu.com/search-duanzufang-p2-0/
http://bj.xiaozhu.com/search-duanzufang-p3-0/
http://bj.xiaozhu.com/search-duanzufang-p4-0/
(2)需要爬取的信息有:标题、地址、价格、房东名称、房东性别和房东头像的链接
"""
# 案例要求:
# 将利用到 Requests和BeautifulSoup库,爬取小猪短租网gz地区租房的信息
#导入相应的库文件
from bs4 import BeautifulSoup as bs
import requests as rs
import time as t
# 加入请求头,伪装成浏览器模式
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134'
}
#定义判断用户性别的函数
def judgment_sex(class_name):
#if class_name==['member_ico1']:
if class_name==["member_girl_ico"]:
return '女'
else:
return '男'
#定义获取详细页URL的函数
def get_links(url):
wb_data=rs.get(url,headers=headers)
soup=bs(wb_data.text,'lxml')
links=soup.select('#page_list > ul > li > a') #links为URL列表
for link in links:
href=link.get("href")
print(href)
get_info(href) #循环出的URL,依次调用get_info()函数
#定义获取网页信息的函数
def get_info(url):
wb_data=rs.get(url,headers=headers)
soup=bs(wb_data.text,'lxml')
#titles=soup.select('span.pr5') #获取title
titles=soup.select('div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
#addresses=soup.select('div.pho_info > h4') #获取地址
addresses=soup.select('div.con_l > div.pho_info > p > span')
prices=soup.select('#pricePart > div.day_l > span')#获取价格
imgs=soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
names=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
#sexs=soup.select('div.js_box.clearfix > div.member_pic > div')
sexs=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')
for title,address,price,img,name,sex in zip(titles,addresses,prices,imgs,names,sexs):
data={
'title':title.get_text().strip(),
'address':address.get_text().strip(),
'price':price.get_text(),
'img':img.get("src"),
'name':name.get_text(),
'sex':judgment_sex(sex.get("class"))
}
print(data) #获取信息并通过字典打印出来
#为程序的主入口’
if __name__=='__main__':
urls=['http://gz.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1,14)]
for singe_url in urls:
get_links(singe_url)
t.sleep(2) #睡眠2秒