数据挖掘项目--数据处理

 

import pandas as pd
from sklearn.model_selection import train_test_split


# 1、导入数据
df = pd.read_csv('data.csv',encoding='gbk') #utf-8格式乱码，设置为gbk才能成功导入

# 2、查看数据类型
print(df.dtypes) #特征太多，这种方式在pycharm上的显示不全，是否有其它方法

# 3、删除无关特征
data = df.drop(['trade_no','bank_card_no','source','id_name','reg_preference_for_trad'],axis=1) #几线城市与预测逾期关系不大？

# 4、删除前后的行列数 
print(df.shape) 
print(data.shape)

# 5、将object类型转换成datatime类型
data['loans_latest_time'] = pd.to_datetime(df['loans_latest_time'],format='%Y-%m-%d %H:%M:%S')
data['latest_query_time'] = pd.to_datetime(df['latest_query_time'],format='%Y-%m-%d %H:%M:%S')

# 6、缺失值处理
# 6.1 查看哪一列含有缺失值
print(data.isnull().any())

# 6.2、尝试用删除含有缺失值的行
cleaned_data = data.dropna()
print(cleaned_data.shape)

# 6.3 含有缺失值的行太多，丢失太多数据，选用填充均值的方式
data = data.fillna(data.mean())

# 6.4查看处理缺失值后的数据
print(data.isnull().any())

# 7、描述统计
print(data.describe())

# 8、训练集、测试集
data['lable'] = data['status']
data = data.drop(['status'],axis=1)

x = data.iloc[1:,2:]
y = data.iloc[1:,-1:]

print(x.shape)
print(y.shape)

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=2018)