import numpy as np
import pandas as pd
import csv
df_data=pd.read_csv(r'E:\数据分析\data\5G\数据集\train_set.csv',encoding='utf-8')
# col_t.csv 是英文字段与汉语备注的对照表
with open(r'E:\数据分析\data\5G\数据集\col_t.csv','r',encoding='gbk',errors='ignore') as csv_data:
reader = csv.reader(csv_data)
# eliminate blank rows if they exist
rows = [row for row in reader if row]
headings = rows[0] # get headings 取第一行
5g_info = {}
for row in rows[1:]:
# append the dataitem to the end of the dictionary entry
# set the default value of [] if this key has not been seen
for col_header, data_column in zip(headings, row):
person_info.setdefault(col_header, []).append(data_column)
#声明2个空列表,使用zip函数将其压缩,并合并为字典形式
d1=[]
d2=[]
for i in 5g_info['字段名称']:
d1.append(i)
for m in 5g_info['中文名称']:
d2.append(m)
d3={}
for i in list(zip(d1,d2)):
d3[i[0]]=i[1]
#重命名df_data的列名
df_data=df_data.rename(columns=d3)
df_data.head()
#计数每个字段的空值,并将其变为DataFrame格式
df_data.isnull().sum()
df_data_sum=pd.DataFrame(df_data.isnull().sum())
df_data_sum=df_data_sum.rename(columns={0:'sum'})
#df_data的行数
df_data.shape[0]
missing_series = df_data.isnull().sum()/df_data.shape[0]
missing_series
#将字段空值率按倒序排列
missing_df = pd.DataFrame(missing_series)
missing_df=missing_df.rename(columns={0:'missing_pct'})
missing_df=missing_df.sort_values(by='missing_pct',ascending=False)
missing_df
#将每个字段的空值数与空值率合并后按倒序排列
joined_data=df_data_sum.join(missing_df,how='left')
joined_data=joined_data.sort_values(by='missing_pct',ascending=False).reset_index(inplace=False).rename(columns={'index':'col'})
joined_data
#计数每行有多少空值
missing_series = df_data.isnull().sum(axis=1)
missing_series
#排除类型不为object的字段
#过滤出不为数字类型的字段
numerical_fea = list(df_data.select_dtypes(exclude=['object']).columns)
obj_fea=list(df_data.select_dtypes(exclude=['int64','float64']).columns) #排除类型不为int、float的字段
category_fea = list(filter(lambda x: x not in numerical_fea, list(df_data.columns)))
def get_numerical_serial_fea(data, feas):
numerical_serial_fea = []
numerical_noserial_fea = []
for fea in feas:
temp = data[fea].nunique()
# 如果同值个数小于10,则认为是类别型数据
if temp <= 10:
numerical_noserial_fea.append(fea)
else:
numerical_serial_fea.append(fea)
return numerical_serial_fea,numerical_noserial_fea
get_numerical_serial_fea(df_data,list(df_data.columns))
#细分市场的非重复元素个数,类似 select distinct
df_data['细分市场'].nunique()