2015-03-19-数据规整化(3)-数据转换

--数据转换
---------------------------------------------------------------------------------------
-----(一)移除重复数据
from pandas import DataFrame,Series
import numpy as np
import pandas as pd
data=DataFrame({'k1':['one']*3+['two']*4,'k2':[1,1,2,3,3,4,4]})
data.duplicated()
data.drop_duplicates()


指定部分列判断
data['v1']=range(7)
data.drop_duplicates(['k1','k2'])
保留最后一个
data.drop_duplicates(['k1','k2'],take_last=True)
---------------------------------------------------------------------------------------
-----(二)利用函数或映射进行数据转换
data=DataFrame({
'food':['bacon','pulled pork','bacon','Pastrami','corned beef','Bacon','pastrami','honey ham','nova lox'],
'ounces':[4,3,12,6,7.5,8,3,5,6]
})
meat_to_animal={
'bacon':'pig',
'pulled pork':'pig',
'pastrami':'cow',
'corned beef':'cow',
'honey ham':'pig',
'nova lox':'salmon'
}
data['animal']=data['food'].map(str.lower).map(meat_to_animal)
data['food'].map(lambda x:meat_to_animal[x.lower()])
---------------------------------------------------------------------------------------
-----(三)替换值
data=Series([1,-999,2,-999,-1000,3])
data.replace(-999,np.nan)
data.replace([-999,-1000],np.nan)
data.replace([-999,-1000],[np.nan,0])
data.replace({-999:np.nan,-1000:0})
---------------------------------------------------------------------------------------
-----(四)重命名轴索引
data=DataFrame(np.arange(12).reshape((3,4)),index=['Ohio','Colorado','New York'],columns=['one','two','three','four'])
data.index.map(str.upper)
data.index=data.index.map(str.upper)
创建数据集的转换版
data.rename(index=str.title,columns=str.upper)
更新部分轴标签
data.rename(index={'OHIO':'INDIANA'},columns={'three':'peekaboo'})
data.rename(index={'OHIO':'INDIANA'},columns={'three':'peekaboo'},inplace=True)
---------------------------------------------------------------------------------------
-----(五)离散化和面元划分
ages=[20,22,25,27,21,23,37,31,61,45,41,32]
bins=[18,25,35,60,100]
cats=pd.cut(ages,bins)
cats.labels
cats.levels
pd.value_counts(cats)
修改闭端
pd.cut(ages,[18,25,35,60,100],right=False)
设置面元名称
group_names=['Youth','YoungAdult','MiddleAged','Senior']
pd.cut(ages,bins,labels=group_names)
等长面元
data=np.random.rand(20)
pd.cut(data,4,precision=2)
得到大小基本相等的面元
data=np.random.randn(1000)
cats=pd.qcut(data,4)
cats
pd.value_counts(cats)
---------------------------------------------------------------------------------------
-----(六)检测和过滤异常值
np.random.seed(12345)
data=DataFrame(np.random.randn(1000,4))
data.describe()
col=data[3]
col[np.abs(col)>3]
data[(np.abs(data)>3).any(1)]
data[np.abs(data)>3]=np.sign(data)*3
---------------------------------------------------------------------------------------
-----(七)排列和随机采样
df=DataFrame(np.arange(5*4).reshape(5,4))
sampler=np.random.permutation(5)
sampler
df
df.take(sampler)
df.take(np.random.permutation(len(df))[:3])


bag=np.array([5,7,-1,6,4])
sampler=np.random.randint(0,len(bag),size=10)
sampler
draws=bag.take(sampler)
draws
---------------------------------------------------------------------------------------
-----(八)计算指标/哑变量
df=DataFrame({'key':['b','b','a','c','a','b'],
'data1':range(6)})
pd.get_dummies(df['key'])


dummies=pd.get_dummies(df['key'],prefix='key')
df_with_dummy=df[['data1']].join(dummies)


mnames=['movie_id','title','genres']
movies=pd.read_table('ch02\movielens\movies.dat',sep='::',header=None,names=mnames)
movies[:10]
genre_iter=(set(x.split('|')) for x in movies.genres)
genres=sorted(set.union(*genre_iter))
dummies=DataFrame(np.zeros((len(movies),len(genres))),columns=genres)
for i,gen in enumerate(movies.genres):
    dummies.ix[i,gen.split('|')]=1
movies_windic=movies.join(dummies.add_prefix('Genre_'))
movies_windic.ix[0]


values=np.random.rand(10)
values
bins=[0,0.2,0.4,0.6,0.8,1]
pd.get_dummies(pd.cut(values,bins))