项目——预处理3

最新推荐文章于 2022-07-15 20:04:36 发布
瓦全
最新推荐文章于 2022-07-15 20:04:36 发布
阅读量221
点赞数
分类专栏：项目
本文链接：https://blog.csdn.net/qq_31150399/article/details/72831923
版权
项目专栏收录该内容
4 篇文章 0 订阅
订阅专栏
20170415
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#预处理

import pandas as pd
import numpy as np
from numpy import NaN

"""
1.删除全重和全空数据
"""
# dateparse = lambda dates: pd.datetime.strptime(dates,'%Y-%m-%d')
# data = pd.read_csv(r'C:\Users\DELL\Desktop\20170515\all.csv',
#                    date_parser=dateparse,
#                    dtype={'EQUIP_ID':object, 'FAULT_TYPE':object, 'INST_DATE':object, 'DETECT_DATE':object,
#                           'FAULT_DATE':object, 'SYNC_ORG_NO':object, 'ORG_NO':object,'ORG_NAME':object,'SORT_CODE':object,
#                           'SPEC_CODE':object, 'COMM_MODE':object, 'ARRIVE_BATCH_NO':object, 'QTY':object,'MANUFACTURER':object})
# print data.head(5)
# print data.info()
# data=data.drop_duplicates()
# data.dropna(how='all')
# data.to_csv(r'C:\Users\DELL\Desktop\20170515\all-1.csv', encoding='utf-8',index=False)

"""
2.在1的基础上选取401-411的智能表
"""
# dateparse = lambda dates: pd.datetime.strptime(dates,'%Y-%m-%d')
# data = pd.read_csv(r'C:\Users\DELL\Desktop\20170515\all-1.csv',
#                    date_parser=dateparse,
#                    dtype={'EQUIP_ID':object, 'FAULT_TYPE':object, 'INST_DATE':object, 'DETECT_DATE':object,
#                           'FAULT_DATE':object, 'SYNC_ORG_NO':object, 'ORG_NO':object,'ORG_NAME':object,'SORT_CODE':object,
#                           'SPEC_CODE':object, 'COMM_MODE':object, 'ARRIVE_BATCH_NO':object, 'QTY':object,'MANUFACTURER':object})
# #去掉空格
# data['EQUIP_ID'] = data['EQUIP_ID'].str.strip()
# data['FAULT_TYPE'] = data['FAULT_TYPE'].str.strip()
# data['INST_DATE'] = data['INST_DATE'].str.strip()
# data['DETECT_DATE'] = data['DETECT_DATE'].str.strip()
# data['FAULT_DATE'] = data['FAULT_DATE'].str.strip()
# data['SYNC_ORG_NO'] = data['SYNC_ORG_NO'].str.strip()
# data['ORG_NO'] = data['ORG_NO'].str.strip()
# data['ORG_NAME'] = data['ORG_NAME'].str.strip()
# data['SORT_CODE'] = data['SORT_CODE'].str.strip()
# data['SPEC_CODE'] = data['SPEC_CODE'].str.strip()
# data['COMM_MODE'] = data['COMM_MODE'].str.strip()
# data['ARRIVE_BATCH_NO'] = data['ARRIVE_BATCH_NO'].str.strip()
# data['QTY'] = data['QTY'].str.strip()
# data['MANUFACTURER'] = data['MANUFACTURER'].str.strip()
#
# # 供电所统一到市级别
# data['ORG_NO_5'] = [x[:5] for x in data['ORG_NO'].values.astype('str')]
#
# # 选出SORT_CODE 为10 的智能表
# data = data[data['SORT_CODE']=='10']
# print data.info()
#
# # 提取故障类型前3位
# print data['FAULT_TYPE'].value_counts()
# data['FAULT_TYPE_1'] = [x[0] for x in data['FAULT_TYPE'].values.astype('str')]
# data['FAULT_TYPE_3'] = [x[0:3] for x in data['FAULT_TYPE'].values.astype('str')]
# print data['FAULT_TYPE'].value_counts()
# print data['FAULT_TYPE_1'].value_counts()
# print data['FAULT_TYPE_3'].value_counts()
#
# # 选取401-411故障
# data = data[data['FAULT_TYPE_1']=='4']
# data = data[(data['FAULT_TYPE_3']!='412')&(data['FAULT_TYPE']!='4')]
# print data['FAULT_TYPE_3'].value_counts()
# print data.head(5)
# print data.info()
#
# #时间处理
# data['FAULT_DATE1'] = pd.to_datetime(data['FAULT_DATE'].str.strip().str.split(' ').str[0])
# data['INST_DATE1'] = pd.to_datetime(data['INST_DATE'].str.strip().str.split(' ').str[0])
# data['DETECT_DATE1'] = pd.to_datetime(data['DETECT_DATE'].str.strip().str.split(' ').str[0])
# # 提取月份
# data['FAULT_MONTH'] = [x.month for x in data['FAULT_DATE1']]
# data['INST_MONTH'] = [x.month for x in data['INST_DATE1']]
# data['FAULT_MONTH'].values.astype('str')
# data['INST_MONTH'].values.astype('str')
# # 计算使用时间
# data['work_days'] = data['FAULT_DATE1'] - data['INST_DATE1']
# data['work_months'] = [x.days/30 if not pd.isnull(x) else np.nan for x in data['work_days']]
# #计算库存时间
# data['save_days'] = data['INST_DATE1'] - data['DETECT_DATE1']
# data['save_months'] = [x.days/30 if not pd.isnull(x) else np.nan for x in data['save_days']]
# #删除数据
# data.drop([ 'EQUIP_ID','FAULT_DATE', 'INST_DATE','DETECT_DATE'],
#          axis=1, inplace=True)
#
# print data['FAULT_TYPE_3'].value_counts()
# print data.head()
# data.to_csv(r'C:\Users\DELL\Desktop\20170515\all-2.csv', encoding='utf-8',index=False)

"""
3.随机取正态分布补全时间
"""

dateparse = lambda dates: pd.datetime.strptime(dates,'%Y-%m-%d')
data = pd.read_csv(r'C:\Users\DELL\Desktop\20170515\all-2.csv',
                   date_parser=dateparse,
                   dtype={ 'FAULT_TYPE':object, 'SYNC_ORG_NO':object, 'ORG_NO':object,'ORG_NAME':object,'SORT_CODE':object,
                          'SPEC_CODE':object, 'COMM_MODE':object, 'ARRIVE_BATCH_NO':object,'QTY': object,'MANUFACTURER':object,'EQUIP_ID':object,
                           'ORG_NO_5': object, 'FAULT_TYPE_1': object, 'FAULT_TYPE_3': object,'FAULT_DATE1': object, 'INST_DATE1': object,
                           'DETECT_DATE1': object, 'FAULT_MONTH': object, 'INST_MONTH': object})
print data.info()
#随机取正态分布补全使用时间
#统计缺失值和小于零的值
count_nan_work_months = data['work_months'].isnull().sum()
print count_nan_work_months
count_0_work_months = len(data['work_months'][data['work_months']<0])
print count_0_work_months
#将小于零的值化为na
data['work_months'][data['work_months']<0]=NaN
print data['work_months']
#再次统计小于零的值
count_0_work_months = len(data['work_months'][data['work_months']<0])
print count_0_work_months
#再次统计空值
count_nan_work_months = data['work_months'].isnull().sum()
print count_nan_work_months
#统计均值和方差
work_months_mean = data['work_months'].mean()
print work_months_mean
work_months_std = data['work_months'].std()
print work_months_std
#补全，生成标准正态分布，大小为空值的大小
rand_1 = np.random.randint(work_months_mean - work_months_std, work_months_mean + work_months_std, size = count_nan_work_months)
data['work_months'][data['work_months'].isnull()] = rand_1
print data['work_months']
# #检查
# a = data['work_months'].isnull().sum()
# print a
# b = len(data['work_months'][data['work_months']<0])
# print b

#随机取正态分布补全库存时间
#统计空值和小于零的值
count_nan_save_months = data['save_months'].isnull().sum()
count_0_save_months = len(data['save_months'][data['save_months']<0])
#将小于零的值化为na
data['save_months'][data['save_months']<0]=NaN
print data['save_months']
#再次统计小于零的值
count_0_save_months = len(data['save_months'][data['save_months']<0])
print count_0_save_months
#再次统计空值
count_nan_save_months = data['save_months'].isnull().sum()
print count_nan_save_months
#统计均值和方差
save_months_mean = data['save_months'].mean()
save_months_std = data['save_months'].std()
#补全
rand_2 = np.random.randint(save_months_mean - save_months_std, save_months_mean + save_months_std, size = count_nan_save_months)
data['save_months'][data['save_months'].isnull()] = rand_2
print data['save_months']
#检查
c = len(data['save_months'][data['save_months']<0])
print c
d = data['save_months'].isnull().sum()
print d

print data.info()
data.to_csv(r'C:\Users\DELL\Desktop\20170515\all-3.csv', encoding='utf-8', index=False)