import requests as req
import time
import pandas as pd
from bs4 import BeautifulSoup
from sqlalchemy import create_engine
import os
global info
#domain = "https://sz.esf.fang.com/"
city = "house/"
def getHouseInfo(url, ipproxies):
info = {}
soup = BeautifulSoup(req.get(url, proxies=ipproxies).text, "html.parser")
resinfo = soup.select(".tab-cont-right .trl-item1")
loca_info = soup.select(".tr-line .trl-item2")
print(url)
#print(resinfo)
#print(loca_info)
# 获取户型、建筑面积、单价、朝向、楼层、装修情况
for re in resinfo:
tmp = re.text.strip().split("\n")
#print(tmp)
name = tmp[1].strip()
#print('*********')
#print(name)
if ("朝向" in name):
name = name.strip("进门")
if ("楼层" in name):
name = name[0:2]
if ("地上层数" in name):
name = "楼层"
if ("装修程度" in name):
name = "装修"
info[name] = tmp[0].strip()
xiaoqu = soup.select(".rcont .blue")[0].text.strip()
info["小区名字"] = xiaoqu
zongjia = soup.select(".tab-cont-right .trl-item")
info["总价"] = zongjia[0].text
info['区'] = soup.select(".rcont .blue")[1].text.strip()
info['镇'] = soup.select(".rcont .blue")[2].text.strip()
#for temp in location:
# print(temp)
print(info)
print("##################")
return info
# 获取总页数
def getTotalPage(domain, city):
res = req.get(domain + city + "i31")
soup = BeautifulSoup(res.text, "html.parser")
endPage = soup.select(".page_al a").pop()['href']
pageNum = endPage.strip("/").split("/")[1].strip("i3")
print("loading.....总共 " + pageNum + " 页数据.....")
return pageNum
# 分页爬取数据
def pageFun(domain, city, i, ipproxies):
pageUrl = domain + city + "i3" + i
print(pageUrl + " loading...第 " + i + " 页数据.....")
res = req.get(pageUrl, proxies=ipproxies)
soup = BeautifulSoup(res.text, "html.parser")
houses = soup.select(".shop_list dl")
pageInfoList = []
for house in houses:
try:
# print(domain + house.select("a")[0]['href'])
info = getHouseInfo(domain + house.select("a")[0]['href'], ipproxies)
pageInfoList.append(info)
print(info)
except Exception as e:
print("---->出现异常,跳过 继续执行", e)
df = pd.DataFrame(pageInfoList)
return df
def makeFile(path):
if not os.path.exists(path):
os.makedirs(path)
def getIPPool():
test_url = 'https://www.baidu.com/'
ip_url = 'http://www.xicidaili.com/'
captcha_12306 = 'https://kyfw.12306.cn/passport/captcha/captcha-image'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64)\
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36', }
ip_response = req.get(ip_url, headers=headers).text
ip_soup = BeautifulSoup(ip_response, 'lxml')
# print(ip_soup)
tag_tr = ip_soup.find_all('tr')
IPPool = []
for tr in tag_tr:
ip_info = {
'ip1': '',
'port2': '',
'location3': '',
'type5': '',
'live_time6': '',
'verify_time7': ''
}
if len(tr.find_all('td')) == 8:
# print(tr.find_all('td')[1].contents[0])
try:
# print(tr.find_all('td'))
ip_info['ip1'] = tr.find_all('td')[1].contents[0]
ip_info['port2'] = tr.find_all('td')[2].contents[0]
ip_info['location3'] = tr.find_all('td')[3].contents[0]
ip_info['type5'] = tr.find_all('td')[5].contents[0]
ip_info['live_time6'] = tr.find_all('td')[6].contents[0]
ip_info['verify_time7'] = tr.find_all('td')[7].contents[0]
IPPool.append(ip_info)
except:
pass
return IPPool
def getIPPool2():
'''
从891网址获取IP池
:return:
'''
test_url = 'https://www.baidu.com/'
IPPool = []
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64)\
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36', }
for i in range(0, 10):
ip_url = 'http://www.89ip.cn/index_' + str(i) + '.html/' #获取前10页url
#ip_url = 'http://www.89ip.cn/index_3.html/'
#headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64)\
#AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36', }
ip_response = req.get(ip_url, headers=headers).text
ip_soup = BeautifulSoup(ip_response, 'lxml')
#print(ip_soup)
tag_tr = ip_soup.find_all('tr')
for tr in tag_tr:
ip_info = {
'ip1': '',
'port2': '',
'location3': '',
'type5': '',
'live_time6': '',
'verify_time7': ''
}
#print(tr)
#print(len(tr))
if len(tr) == 11:
print("#################")
if (not tr.find_all('td')):
continue
print(tr.find_all('td')[0].contents[0])
try:
# print(tr.find_all('td'))
ip_info['ip1'] = tr.find_all('td')[0].contents[0].split('\t')[3]
ip_info['port2'] = tr.find_all('td')[1].contents[0].split('\t')[3]
ip_info['location3'] = tr.find_all('td')[2].contents[0]
ip_info['type5'] = 'noType'
ip_info['live_time6'] = tr.find_all('td')[4].contents[0]
ip_info['verify_time7'] = tr.find_all('td')[4].contents[0]
IPPool.append(ip_info)
except:
pass
print(IPPool)
return IPPool
def writePriceToData(domain, city):
'''
获取房屋的信息并存入CSV
:param domain: 房产地址
:param city:
:return:
'''
timeFormat = time.strftime('%Y%m%d %H%M%S',time.localtime())
path = 'F:\\python\\housePrice'
makeFile(path)
fileName = path + '\\'+ domain.split('.')[0].split('/')[-1] + timeFormat + '.csv'
all_Price = pd.DataFrame()
if len(getIPPool()) != 0:
ip_pool = getIPPool()
print('使用西刺代理!')
elif len(getIPPool2()) != 0:
ip_pool = getIPPool2()
print('使用891代理!')
else:
print('获取代理IP失败 ,请检查代理网址是否可用 !')
return -1
print(ip_pool)
for i in range(1, int(getTotalPage(domain, city)) + 1):
for ip in ip_pool:
ipProxies = {
'http': 'https://' + ip['ip1']
}
print(ipProxies)
try:
df_onePage = pageFun(domain, city, str(i), ipProxies)
all_Price = all_Price.append(df_onePage)
break
except Exception as e:
continue
print("Exception", e)
all_Price.to_csv(fileName, encoding='utf-8-sig')
#pd.io.sql.to_sql(df_onePage, "city_house_price", connect, schema="houseinfo", if_exists="append")
return 0
domains = ['https://sz.esf.fang.com/', 'https://gz.esf.fang.com/']
#domains = ['https://sh.esf.fang.com/', 'https://hz.esf.fang.com/', 'https://zz.esf.fang.com/', 'https://zhumadian.esf.fang.com/', 'https://xinyang.esf.fang.com/']
for domain in domains:
print(domain)
writePriceToData(domain, city)