2024年最全小牧用Python 爬取数万条房产数据，揭秘一线城市生存压力有多大

最新推荐文章于 2024-07-28 10:15:21 发布

2401_84562251

最新推荐文章于 2024-07-28 10:15:21 发布

阅读量619

点赞数 20

分类专栏：程序员文章标签： python 开发语言 windows

本文链接：https://blog.csdn.net/2401_84562251/article/details/138383933

版权

程序员专栏收录该内容

192 篇文章 0 订阅

订阅专栏

for div in divs:

ps = div.find_all(“p”)

try: # 捕获异常，因为页面中有些数据没有被填写完整，或者被插入了一条广告，则会没有相应的标签，所以会报错

for index, p in enumerate(ps): # 从源码中可以看出，每一条 p 标签都有我们想要的信息，故在此遍历 p 标签，

text = p.text.strip()

print(text) # 输出看看是否为我们想要的信息

print(“===================================”)

爬取并存进 MongoDB 数据库

roomMsg = ps[1].text.split(“|”)

rentMsg 这样处理是因为有些信息未填写完整，导致对象报空

area = roomMsg[2].strip()[:len(roomMsg[2]) - 2]

rentMsg = self.getRentMsg(

ps[0].text.strip(),

roomMsg[1].strip(),

int(float(area)),

int(ps[len(ps) - 1].text.strip()[:len(ps[len(ps) - 1].text.strip()) - 3]),

ps[2].text.strip(),

ps[3].text.strip(),

ps[2].text.strip()[:2],

roomMsg[3],

)

rent.insert(rentMsg)

except:

continue

数据分析：

求一个区的房租单价（平方米/元）

def getAvgPrice(self, region):

areaPinYin = self.getPinyin(region=region)

collection = self.zfdb[areaPinYin]

totalPrice = collection.aggregate([{‘ $KaTeX parse error: Expected '}', got 'EOF' at end of input: …oup': {'_id': '$ region’, ‘total_price’: {‘ $s u m^{'} :^{'}$ price’}}}])

totalArea = collection.aggregate([{‘ $KaTeX parse error: Expected '}', got 'EOF' at end of input: …oup': {'_id': '$ region’, ‘total_area’: {‘ $s u m^{'} :^{'}$ area’}}}])

totalPrice2 = list(totalPrice)[0][“total_price”]

totalArea2 = list(totalArea)[0][“total_area”]

return totalPrice2 / totalArea2

获取各个区每个月一平方米需要多少钱

def getTotalAvgPrice(self):

totalAvgPriceList = []

totalAvgPriceDirList = []

for index, region in enumerate(self.getAreaList()):

avgPrice = self.getAvgPrice(region)

totalAvgPriceList.append(round(avgPrice, 3))

totalAvgPriceDirList.append({“value”: round(avgPrice, 3), “name”: region + " " + str(round(avgPrice, 3))})

return totalAvgPriceDirList

获取各个区每一天一平方米需要多少钱

def getTotalAvgPricePerDay(self):

totalAvgPriceList = []

for index, region in enumerate(self.getAreaList()):

avgPrice = self.getAvgPrice(region)

totalAvgPriceList.append(round(avgPrice / 30, 3))

return (self.getAreaList(), totalAvgPriceList)

获取各区统计样本数量

def getAnalycisNum(self):

analycisList = []

for index, region in enumerate(self.getAreaList()):

collection = self.zfdb[self.pinyinDir[region]]

print(region)

totalNum = collection.aggregate([{‘ $KaTeX parse error: Expected '}', got 'EOF' at end of input: …'total_num': {'$ sum’: 1}}}])

totalNum2 = list(totalNum)[0][“total_num”]

analycisList.append(totalNum2)

return (self.getAreaList(), analycisList)

获取各个区的房源比重

def getAreaWeight(self):

result = self.zfdb.rent.aggregate([{‘ $KaTeX parse error: Expected '}', got 'EOF' at end of input: …oup': {'_id': '$ region’, ‘weight’: {‘$sum’: 1}}}])

areaName = []

areaWeight = []

for item in result:

if item[“_id”] in self.getAreaList():

areaWeight.append(item[“weight”])

areaName.append(item[“_id”])

print(item[“_id”])

print(item[“weight”])

print(type(item))

return (areaName, areaWeight)

获取 title 数据，用于构建词云

def getTitle(self):

collection = self.zfdb[“rent”]

queryArgs = {}

projectionFields = {‘_id’: False, ‘title’: True} # 用字典指定需要的字段

searchRes = collection.find(queryArgs, projection=projectionFields).limit(1000)

content = ‘’

for result in searchRes:

print(result[“title”])

content += result[“title”]

return content

获取户型数据（例如：3 室 2 厅）

def getRooms(self):

results = self.zfdb.rent.aggregate([{‘ $KaTeX parse error: Expected '}', got 'EOF' at end of input: …oup': {'_id': '$ rooms’, ‘weight’: {‘$sum’: 1}}}])

roomList = []

weightList = []

for result in results:

roomList.append(result[“_id”])

weightList.append(result[“weight”])

print(list(result))

return (roomList, weightList)

获取租房面积

def getAcreage(self):

results0_30 = self.zfdb.rent.aggregate([

{‘ $KaTeX parse error: Expected '}', got 'EOF' at end of input: \dotsh': {'area': {'$ gt’: 0, ‘$lte’: 30}}},