# -*- coding: utf-8 -*-
# @Time : 2017/9/5 10:58
# @Author : z
# @File : 房天下.py
# @Software: PyCharm
import requests
from bs4 import BeautifulSoup
import re
import threadpool
class House(object):
def __init__(self):
pass
def get_url(self,url):
response = requests.get(url)
response.encoding = response.apparent_encoding
self.html = response.text
#解析数据,拼接初始url
def join_url(self):
# url = 'http://js.soufunimg.com/homepage/new/family/css/citys.js?v=20170520'
# response=eval(requests.get(url).text[14:-1])
# for ur in response:
# self.gain_num(ur['name'],'http://newhouse.'+ur['url'].split('.')[0].split('//')[1]+'.fang.com/house/s/')
self.gain_num('上海', 'http://newhouse.sh.fang.com/house/s/')
#获取最大数。循环出新url
def gain_num(self, city, city_url):
self.get_url(city_url)
max_num =re.findall('\d+',re.findall(re.compile('<span class="ff3333">.*?</span>/(.*?)</span>', re.S), self.html)[0])[0]
for num in range(1,int(max_num)+1):
# print('++++++++++++++++++++++++++++++++++++++{}++++++++++++++++++++++++++++++++++++++++++++++'.format(num))
url = city_url+'b9'+str(num)
self.parse_page(city,url,num)
#xianccccccccccc
# list = [city_url + 'b9' + str(num) for num in range(1, int(1) + 1)
# # url = city_url+'b9'+str(num)
#
# thind = threadpool.ThreadPool
# t = threadpool.makeRequests(self.parse_page, (city, list))
#解析首页,获取城市名,
def parse_page(self,city,city_url,num):
self.get_url(city_url)
soup = BeautifulSoup(self.html,'lxml')
for con in soup.find('div',id='newhouse_loupai_list').find('ul').find_all('li'):
house =con.find('div','nlc_details').find('div','nlcd_name')
price = con.find('div', 'nlc_details').find('div', 'nhouse_price')
district_name =con.find('div','nlc_details').find('div','address').get_text().strip().split(']')[0].replace('\t','').replace('[','')
house_url = house.a['href']
project_name= house.a.get_text().strip()
if price:
project_price=price.find('span').get_text()
self.parse_detail(house_url,city,district_name,project_name,project_price,num)
def parse_detail(self,city_url,city_name,district_name,project_name,project_price,num):
self.get_url(city_url)
detail = BeautifulSoup(self.html,'lxml').find('div',id='orginalNaviBox').find_all('a')[1]['href']
self.get_url(detail)
soup = BeautifulSoup(self.html,'lxml')
#评分
# grade = soup.find('div','main-info-comment').find_all('span')[2].get_text()
# #点评数
# num = soup.find('div','main-info-comment').find_all('span')[3].get_text()[1:-1]
content = soup.find_all('div', 'main-item')
#基本信息
print('------------------------------------1-----------------------------------------------')
print('***********************')
print('第{}页'.format(num))
print(city_name)
print(project_name)
print(district_name)
print(project_price)
print('***********************')
clearfix=content[0].find('ul','list clearfix').find_all('li')
project_type =clearfix[0].get_text().strip().replace('\n','').replace('\t','')
project_point =','.join([te.get_text() for te in clearfix[1].find('div','list-right').find_all('span','tag')])
project_buildform =clearfix[2].get_text().replace('\n','').replace('\t','')
project_fixture =clearfix[3].get_text().strip().replace('\n','').replace('\t','')
project_ownlife =clearfix[4].get_text().strip().replace('\n','').replace('\t','')
line_name =clearfix[5].get_text().strip().replace('\n','').replace('\t','')
developer_name =clearfix[6].get_text().strip().replace('\n','').replace('\t','')
project_address =clearfix[7].get_text().strip().replace('\n','').replace('\t','')
print(project_type)
print('项目特色:'+project_point)
print(project_buildform)
print(project_fixture)
print(project_ownlife)
print(line_name)
print(developer_name)
print(project_address)
print('')
#销售信息
list1 =content[1].find('ul','list clearfix').find_all('li')
table = content[1].find('div','main-table')
project_salestatu =list1[0].get_text().replace('\n','').replace('\t','').strip()
project_discount =list1[1].get_text().replace('\n','').replace('\t','').strip()
project_opendate=list1[2].get_text().replace('\n','').replace('\t','').strip()
project_getdate=list1[3].get_text().replace('\n','').replace('\t','').strip()
project_office=list1[4].get_text().replace('\n','').replace('\t','').strip()
project_roomclass=list1[6].get_text().replace('\n','').replace('\t','').strip()
print(project_salestatu)
print(project_discount)
print(project_opendate)
print(project_getdate)
print(project_roomclass)
# print('-------------1-------------------')
if table:
for i in table.find_all('table'):
for td in i.find_all('tr')[1:]:
presale_name1 = td.find_all('td')[0].get_text()
presale_date1 = td.find_all('td')[1].get_text()
presale_build = td.find_all('td')[2].get_text()
print('预售许可证: '+presale_name1)
print('发证时间: '+presale_date1)
print('绑定楼栋: '+presale_build)
print(' ')
#配套信息
try:
project_traffic = content[2].find('div','set').get_text().replace('\n','').replace('\br','').replace(' ','').strip()
project_support = content[2].find('div','set bd-1').find('p').get_text().replace('\n','').replace('\br','').replace('\t','').strip()
print('交通配套:' + project_traffic)
print('项目配套:' + project_support)
except:
pass
print(' ')
#小区规划
project = content[3].find('ul','clearfix list').find_all('li')
project_allarea = project[0].get_text().replace('\n','').replace('\t','').strip()
project_buildarea = project[1].get_text().replace('\n','').replace('\t','').strip()
project_volume = project[2].get_text().replace('\n','').replace('\t','').strip()
project_green = project[3].get_text().replace('\n','').replace('\t','').strip()
project_carpark = project[4].get_text().replace('\n','').replace('\t','').strip()
build_number = project[5].get_text().replace('\n','').replace('\t','').strip()
project_household = project[6].get_text().replace('\n','').replace('\t','').strip()
manager_name = project[7].get_text().replace('\n','').replace('\t','').strip()
manager_price = project[8].get_text().replace('\n','').replace('\t','').strip()
build_floor = project[9].get_text().replace('\n','').replace('\t','').strip()
print(project_allarea)
print(project_buildarea)
print(project_volume)
print(project_green)
print(build_number)
print(project_household)
print(manager_name)
print(manager_price)
print(build_floor)
print(' ')
#价格信息
price = content[4].find_all('table')
for tr in price:
for td in tr.find_all('tr')[1:]:
# print(td)
price_record = td.find_all('td')[0].get_text()
price_value = td.find_all('td')[1].get_text()
price_low = td.find_all('td')[2].get_text()
price_caption = td.find_all('td')[3].get_text()
print('记录时间:'+price_record)
print('均价:'+price_value)
print('起价:'+price_low)
print('价格描述:'+price_caption)
print(' ')
#项目简介
project_caption = content[5].find('p','intro').get_text().strip()
print('项目简介:'+project_caption)
print(' ')
print('---------------------------------------2-------------------------------------------')
print(' ')
if __name__ == '__main__':
House().join_url()
(代码有点瑕疵,,如果有更好的解决方式,,联系我!!共同进步!!)
代码