python实战计划_#python实战计划#1.3练习项目

抓取结果

p{'lorder_gender': '女', 'prices': '438', 'lorder_name': 'gjy1234', 'addr': '上海市徐汇区高安路69弄', 'house_image': 'http://image.xiaozhustatic1.com/00,800,533/7,0,86,9803,1800,1200,0b2f41ad.jpg', 'title': '衡山路 精致一居 近地铁1/7/9/10号线'}

{'lorder_gender': '男', 'prices': '538', 'lorder_name': '王老闆', 'addr': '上海市虹口区北苏州路400号河滨大楼', 'house_image': 'http://image.xiaozhustatic1.com/00,800,533/4,0,68,8498,1798,1200,ca0c88ef.jpg', 'title': '看外滩的超大阳台房 沙逊1935的河滨大楼'}

{'lorder_gender': '男', 'prices': '998', 'lorder_name': '风子上海', 'addr': '上海市徐汇区余庆路', 'house_image': 'http://image.xiaozhustatic1.com/00,801,533/6,0,59,6286,1136,756,d7287d22.jpg', 'title': '【春上春宿】法租界名人故居/有松鼠飞鸟的院子'}

……

CODE

from bs4 import BeautifulSoup

import requests

import time

urls = ["http://sh.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,3)]

def get_detail_url(url):

web_data = requests.get(url)

time.sleep(1)

soup = BeautifulSoup(web_data.text, 'lxml')

detail_urls = soup.select('#page_list > ul > li >div.result_btm_con.lodgeunitname')

for single_detail_url in detail_urls:

data = single_detail_url.get('detailurl')

get_page_details(data)#获取每页详细内容

def get_page_details(url, infos = None):

web_data = requests.get(url)

time.sleep(1)

soup = BeautifulSoup(web_data.text, 'lxml')

titles =soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em ')

addrs = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5 ')

prices = soup.select('div.day_l > span ')

house_images = soup.select('#curBigImage')

lorder_names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')

lorder_genders = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span ')

if infos == None:

for title, addr, price, house_image, lorder_name, lorder_gender in zip(titles, addrs, prices, house_images, lorder_names, lorder_genders):

info = {

'title':title.get_text(),

'addr':addr.get_text().strip(),

'prices':price.get_text(),

'house_image':house_image.get('src'),

'lorder_name':lorder_name.get('title'),

'lorder_gender':get_gender(lorder_gender.get('class'))

}

print(info)

def get_gender(class_name):

if class_name ==['member_boy_ico']:

return '男'

elif class_name == ['member_girl_ico']:

return '女'

else:

return None

for url in urls:

get_detail_url(url)

心得体会：

这节课安装lxml库的时候被难住了，网上到处搜，最后发现解决办法：

Paste_Image.png

函数可以嵌套使用，在本节课的练习中使用了三个函数，用get_detail_url()函数嵌套get_page_details()。

图片的抓取是难点，本例中还没有熟练掌握。抓取结果疑似无法下载。