北京链家二手房数据分析

首先我们需要将链家二手房的数据抓取下来，用我们上课学的内容，很容易实现（bs4,re,urllib，搞定！）：

import re
import csv
from bs4 import BeautifulSoup
from urllib import request

# 成功打开页面时返回页面对象，否则打印错误信息，退出程序
def get_bsobj(url):
## 获取 html 页面
    headers = {"Accept-Language":"zh-CN,zh;q=0.9",
               "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"}

    req = request.Request(url,
                      headers=headers)

## 获取 html 页面
    html = request.urlopen(req).read().decode()
    bsobj = BeautifulSoup(html, "html5lib")
    if bsobj:
        return bsobj
    else:
        print("页面错误")
        return None


# 将页面中每一条房屋信息保存为一个字典，将所有的字典保存在列表中，返回列表
def get_house_info_list(url):
    house_info_list = []
    bsobj = get_bsobj(url)
    if not bsobj:
        return None

    house_list = bsobj.find_all("li", {"class":"clear"})
    print(len(house_list))
    for house in house_list:
        try:
            # 标题
            title = house.find("div", {"class": "title"}).get_text()
            #print(title)
    
            # 获取信息数据（例：加怡名城 | 2室1厅 | 62.48平米 | 西 | 精装），通过“|”符号分割字符串
            info = house.find("div", {"class": "houseInfo"}).get_text().split("/")
            #print(info)
            
            # 小区（例：加怡名城），strip()去除字符串两边的空格，encode，将字符串编码成 utf-8 格式
            block = info[0].strip()
            #print(block)
    
            # 房型（例：2室一厅）
            house_type = info[1].strip()
            #print(house_type)
    
            # 面积大小，保留整数（例：62.48平米，保留整数后为 62）
            size_info = info[2].strip()
            size = re.findall(r"\d+", size_info)[0]
            #print(size)
    
            # 价格，保留整数（例：120.3万，保留整数后为 120）
            price_info = house.find("div", {"class": "totalPrice"}).span.get_text()
            price = re.findall(r"\d+", price_info)[0]
            #print(price)
            
            # 添加到列表中
            house_info_list.append({
                "title": title,
                "price": int(price),
                "size": int(size),
                "block": block,
                "house_type": house_type
            })
            
        except IndexError:
            pass

    #print(house_info_list)
    return house_info_list

# 读取前三个页面的房屋信息，将信息保存到 house.csv 文件中
def house(url):
    house_info_list = []

    # range(10),即前10个子页面  #这里只抓取前10个页面
    for i in range(10):
        new_url = url + 'pg' + str(i+1)
        house_info_list.extend(get_house_info_list(new_url))
        print(new_url)

    if house_info_list:
        # 将数据保存到 house.csv 文件中
        with open("./house.csv", "w+") as f:
            # writer 对象，修改默认分隔符为 "|"
            writer = csv.writer(f, delimiter="|")
            for house_info in house_info_list:
                title = house_info.get("title")
                price = house_info.get("price")
                size = house_info.get("size")
                block = house_info.get("block")
                house_type = house_info.get("house_type")
                # 写入一行

                try:
                     writer.writerow([title, int(price), int(size), block, house_type])
                     print(block, price, size)

                except:

                      continue

house("https://bj.lianjia.com/ershoufang/")

 

# 简单数据分析案例，太简单了，直接上代码：这里统计了房屋的面积，价格的平均值，标准差，并用直方图来显示结果
 

# -*- coding: utf-8 -*-
"""
Created on Sun Jun 10 13:07:02 2018

@author: Jun
"""

import numpy
import matplotlib.pyplot as plt

# 读取 house.csv 文件中价格和面积列
price, size = numpy.loadtxt('house.csv', delimiter='|', usecols=(1, 2), unpack=True)

# 求价格和面积的平均值
price_mean = numpy.mean(price)
size_mean = numpy.mean(size)
print("平均价格为：(万元)", price_mean)
print("平均面积为：(平方米)", size_mean)

# 求价格和面积的方差
#price_var = numpy.var(price)
price_var = numpy.std(price)
#size_var = numpy.var(size)
size_var = numpy.std(size)
print("价格的标准差为：(万元)", price_var)
print("面积的标准差为：", size_var)

price, size = numpy.loadtxt('house.csv', delimiter='|', usecols=(1, 2), unpack=True)
plt.figure()
plt.subplot(211)
plt.title("/10000RMB")
plt.hist(price, bins=20)

plt.subplot(212)
plt.xlabel("/m**2")

plt.hist(size, bins=20)

# 运行结果截图