《Hands-on Machine Learning with Scikit-Learn and TensorFlow》读书笔记

最新推荐文章于 2024-08-13 08:48:54 发布

ju22

最新推荐文章于 2024-08-13 08:48:54 发布

阅读量3.4k

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/juzexia/article/details/79096354

版权

python 专栏收录该内容

19 篇文章 1 订阅

订阅专栏

这篇博客是《Hands-on Machine Learning with Scikit-Learn and TensorFlow》的读书笔记，重点介绍了数据预处理的步骤。内容包括读取CSV数据，绘制散点图，进行数据分割，处理缺失值（如使用中位数填充），对分类变量进行编码，以及使用Pipeline和FeatureUnion结合连续和分类特征进行标准化。预处理过程中涉及到的数据集包括地理位置、人口、房价等属性。

摘要由CSDN通过智能技术生成

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt

housing=pd.read_csv('D:\\pythondata\\housing.csv') 

housing.plot(kind="scatter", x="longitude", y="latitude")

housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.1)

housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4,
s=housing["population"]/100, label="population",
c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True,
)


plt.legend()


housing["income_cat"] = np.ceil(housing["median_income"] / 1.5)

housing["income_cat"].where(housing["income_cat"] < 5, 5.0, inplace=True)

from sklearn.model_selection import StratifiedShuffleSplit

split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)

for train_index,test_index in split.split(housing,housing["income_cat"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]
    
    
housing["income_cat"].value_counts() / len(housing)
    

#删除income_cat  使得数据回到原始模样

for set in (strat_train_set, strat_test_set):
    set.drop(["income_cat"], axis=1, inplace=True)    



corr_matrix = housing.corr()

corr_matrix["median_house_value"].sort_values(ascending=False)


# 下面做数据清洗

housing = strat_train_set.drop("median_house_value", axis=1)
housing_labels = strat_train_set["median_house_value"].copy()


#对缺失值的处理
housing.dropna(subset=["total_bedrooms"])    # option 1   将缺失值删除

housing.drop("total_bedrooms", axis=1) # option 2   将缺失值删除

median = housing["total_bedrooms"].median()
housing["total_bedrooms"].fillna(median) # option 3    用中位数填充缺失值

#sklearn 库提供了一些可以用于处理缺失值的模块

from sklearn.preprocessing import Imputer   #导入模块

imputer = Imputer(strategy="median")      

housing_num = housing.drop("ocean_proximity",axis=1)

imputer.fit(housing_num)

imputer.statistics_          #查看各个变量用于填充的中位数的值

housing_num.median().values

X = imputer.transform(housing_num)

housing_tr = pd.DataFrame(X,columns=housing_num.columns)


#对于分类变量的处理，将其转换为数字形式

from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()
housing_cat = housing["ocean_proximity"]
housing_cat_encoded = encoder.fit_transform(housing_cat)
housing_cat_encoded

print encoder.classes_



from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room = True):      # no *args or **kargs
        self.add_bedrooms_per_room = add_bedrooms_per_room
    
    def fit(self, X, y=None):
        return self # nothing else to do
    
    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
        population_per_household = X[:, population_ix] / X[:, household_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household,bedrooms_per_room]
        else:
           return np.c_[X, rooms_per_household, population_per_household]


attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)




from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
num_pipeline = Pipeline([
('imputer', Imputer(strategy="median")),
('attribs_adder', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
    
housing_num_tr = num_pipeline.fit_transform(housing_num)


#由于数据中既有连续型变量，又有分类变量，要分开处理，因此用num_pipeline 和 cat_pipeline两个管道分析处理，
# 然后通过FeatureUnion 函数合并。


from sklearn.pipeline import FeatureUnion
num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]
num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)),
        ('imputer', Imputer(strategy="median")),
        ('attribs_adder', CombinedAttributesAdder()),
        ('std_scaler', StandardScaler()),
    ])
    
cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),
        ('label_binarizer', LabelBinarizer()),
    ])

full_pipeline = FeatureUnion(transformer_list=[
        ("num_pipeline", num_pipeline),
        ("cat_pipeline", cat_pipeline),
    ])


housing_prepared = full_pipeline.fit_transform(housing)    #调用合并后的管道函数