端到端机器学习项目

import os
import tarfile
from six.moves import urllib
DOWNLOAD_ROOT=“https://raw.githubusercontent.com/ageron/handson-ml/master/”
HOUSING_PATH=“datasets/housing”
HOUSING_URL=DOWNLOAD_ROOT+HOUSING_PATH+"/housing.csv"
def fetch_housing_data(housing_url=HOUSING_URL,housing_path=HOUSING_PATH):
if not os.path.isdir(housing_path):
os.makedirs(housing_path)
tgz_path=os.path.join(housing_path,“housing.csv”)
urllib.request.urlretrieve(housing_url,tgz_path)

import pandas as pd
def load_housing_data(housing_path=HOUSING_PATH):
csv_path=os.path.join(housing_path,“housing.csv”)
return pd.read_csv(csv_path)
load_data=load_housing_data()
load_data.head()
load_data.info()
load_data[‘ocean_proximity’].value_counts()
load_data.describe()
<class ‘pandas.core.frame.DataFrame’>
RangeIndex: 20640 entries, 0 to 20639
Data columns (total 10 columns):
longitude 20640 non-null float64
latitude 20640 non-null float64
housing_median_age 20640 non-null float64
total_rooms 20640 non-null float64
total_bedrooms 20433 non-null float64
population 20640 non-null float64
households 20640 non-null float64
median_income 20640 non-null float64
median_house_value 20640 non-null float64
ocean_proximity 20640 non-null object
dtypes: float64(9), object(1)
memory usage: 1.6+ MB
longitude latitude housing_median_age total_rooms total_bedrooms population households median_income median_house_value
count 20640.000000 20640.000000 20640.000000 20640.000000 20433.000000 20640.000000 20640.000000 20640.000000 20640.000000
mean -119.569704 35.631861 28.639486 2635.763081 537.870553 1425.476744 499.539680 3.870671 206855.816909
std 2.003532 2.135952 12.585558 2181.615252 421.385070 1132.462122 382.329753 1.899822 115395.615874
min -124.350000 32.540000 1.000000 2.000000 1.000000 3.000000 1.000000 0.499900 14999.000000
25% -121.800000 33.930000 18.000000 1447.750000 296.000000 787.000000 280.000000 2.563400 119600.000000
50% -118.490000 34.260000 29.000000 2127.000000 435.000000 1166.000000 409.000000 3.534800 179700.000000
75% -118.010000 37.710000 37.000000 3148.000000 647.000000 1725.000000 605.000000 4.743250 264725.000000
max -114.310000 41.950000 52.000000 39320.000000 6445.000000 35682.000000 6082.000000 15.000100 500001.000000

import matplotlib.pyplot as plt
load_data.hist(stacked=True, bins=100, figsize=(20,15),alpha=0.5)
plt.show()

import hashlib
import numpy as np
def test_set_check(identifier,test_ratio,hash):
return hash(np.int64(identifier)).digest()[-1]<256*test_ratio
def split_train_test_by_id(data,test_ratio,id_column,hash=hashlib.md5):
ids=data[id_column]
in_test_set=ids.apply(lambda id_:test_set_check(id_,test_ratio,hash))
return data.loc[~in_test_set],data.loc[in_test_set]
housing_with_id=load_data.reset_index()
train_set,test_set=split_train_test_by_id(housing_with_id,0.2,“index”)
函数说明
hash.digest()返回摘要，作为二进制数据字符串值
np.int64()强制转换成int64类型，和int不是一个类型
apply()将函数应用到由各列或行形成的数组上
lambda匿名函数
loc通过标签索引行数据