Python数据分析基础——pandas用法

pd.Series

import pandas as pd
s=pd.Series([1,3,6,np.nan,44,1])
print(s)

生成数组

dates=pd.date_range('20190101',periods=6)
print(dates)
df=pd.DataFrame(np.random.randn(6,4),index=dates,columns=['a','b','c','d'])
# df=pd.DataFrame(np.arange(24).reshape(6,4),index=dates,columns=['a','b','c','d'])
print(df)

df1=pd.DataFrame(np.arange(12).reshape(3,4))
print(df1)

调用内部参数

df2=pd.DataFrame({'A':1.,
                  'B':pd.Timestamp('20130102'),
                  'C':pd.Series(1,index=list(range(4)),dtype='float32'),
                  'D':np.array([3]*4,dtype='int32'),
                  'E':pd.Categorical(['test','train','test','train']),
                  'F':'foo'})
print(df2)
print(df2.dtypes)
print(df2.index)
print(df2.columns)
print(df2.values)
print(df2.describe())

转置

print(df2.T)

排序

print(df2.sort_index(axis=1,ascending=False))
print(df2.sort_index(axis=0,ascending=False))
print(df2.sort_values(by='E'))
#降序排列
df2=df.sort_values('total_confirm',ascending=False)

操作行列及元素

选取

dates=pd.Timestamp('20130102')
dates=pd.date_range('20130101',periods=6)
print(dates)
# df=pd.DataFrame(np.random.randn(6,4),index=dates,columns=['a','b','c','d'])
df=pd.DataFrame(np.arange(24).reshape(6,4),index=dates,columns=['a','b','c','d'])
print(df)
print(df['a'])
print(df.a)
print(df[0:3])
print(df['20130102':'20130104'])
print(df.loc['20130102'])
print(df.loc['20130102',['a','b']])
print(df['20130102':'20130104'，['a','b']])
print(df[0:3，['a','b']])
print(df.iloc[3:5,1])
print(df.iloc[0:3,-6:-2])
print(df.iloc[[1,3,5],1:3])
print(df.ix[:3,['b','c']])
print(df[df.a<8])

替换

dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape(6,4),index=dates,columns=['a','b','c','d'])
print(df)
df.iloc[5,:]=[3,3,3,3]
print(df)
df.loc['20130101','b']=8888
print(df)
df[df.a>4]=0
df.a[df.a>4]=0
print(df)

增加

##增加列
df['F']=np.nan
df['E']=pd.Series([1,2,3,4,5,6],index=pd.date_range('20130102',periods=6))
df['G']=np.array(range(6))
df['H']=np.arange(6)
print(df)
#增加行
df.loc[6]=np.arange(4)
print(df)

复制并修改行列标签

dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape(6,4),index=dates,columns=['a','b','c','d'])
print(df)
df=pd.DataFrame(np.arange(4).reshape(4,1),columns=['a','b','c','d'])
df.iloc[5,:4]=[1,2,3,4]
#复制并修改行列标签
df2=df.reindex(index=range(6))
# df2=df.reindex(index=[1,2,3,4,5,6],columns=[1,2,3,4])
# df2=df.reset_index(drop=True)#重新设置行索引
print(df2)

空值处理

print(df.dropna(axis=0,how='any'))
print(df.dropna(axis=0,how='all'))
print(df.dropna(axis=1,how='any'))
print(df.fillna(value=0))
print(df.isnull())
print(np.any(df.isnull())==True)

横向合并merge

left=pd.DataFrame({'key':['K0','K1','K2','K3'],
                   'A':['A0','A1','A2','A3'],
                   'B': ['B0', 'B1', 'B2', 'B3']})
right=pd.DataFrame({'key':['K0','K1','K2','K3'],
                    'C': ['C0', 'C1', 'C2', 'C3'],
                    'D': ['D0', 'D1', 'D2', 'D3'],})
print(left)
print(right)
res=pd.merge(left,right,on='key')
print(res)

left=pd.DataFrame({'key1':['K0','K0','K1','K2'],
                   'key2':['K0','K1','K0','K1'],
                   'A':['A0','A1','A2','A3'],
                   'B': ['B0', 'B1', 'B2', 'B3']})
right=pd.DataFrame({'key1':['K0','K1','K1','K2'],
                    'key2':['K0','K0','K0','K0'],
                    'C': ['C0', 'C1', 'C2', 'C3'],
                    'D': ['D0', 'D1', 'D2', 'D3'],})
print(left)
print(right)
res=pd.merge(left,right,on=['key1','key2'],how='outer')###how="left","right","outer"
print(res)

df1=pd.DataFrame({'col1':[0,1],'col_left':['a','b']})
df2=pd.DataFrame({'col1':[1,2,2],'col_right':[2,2,2]})
print(df1)
print(df2)
res1=pd.merge(df1,df2,on='col1',how='outer',indicator=True)
res2=pd.merge(df1,df2,on='col1',how='outer',indicator='indicator_column')
print(res1)
print(res2)

left=pd.DataFrame({ 'A':['A0','A1','A2'],
                   'B': ['B0', 'B1', 'B2']},
                    index=['K0','K1','K2'])
right=pd.DataFrame({ 'C': ['C0', 'C1', 'C2',],
                     'D': ['D0', 'D1', 'D2']},
                    index=['K0', 'K2', 'K3'])
print(left)
print(right)
res=pd.merge(left,right,left_index=True,right_index=True,how='outer')
# res=pd.merge(left,right,left_index=True,right_index=True,how='inner')
print(res)

boys=pd.DataFrame({'k':['K0','K1','K2'],'age':[1,2,3]})
girls=pd.DataFrame({'k':['K0','K1','K2'],'age':[4,5,6]})
print(boys)
print(girls)
res=pd.merge(boys,girls,on='k',suffixes=['_boys','_girls'],how='inner')
print(res)

或者

df1=pd.DataFrame(np.ones((10,2)))
df2=pd.DataFrame(np.zeros((8,2)))
pd.concat([df1,df2],axis=1)，不全是的数据默认NaN

纵向合并

df1=pd.DataFrame(np.ones((10,2)))
df2=pd.DataFrame(np.zeros((8,2)))
pd.concat([df1,df2])

分组计算group by

import pandas as pd
import numpy as np
df = pd.DataFrame({'key1':['a','a','b','b','a'],
                   'key2':['one','two','one','two','one'],
                   'data1':np.random.randn(5),
                   'data2':np.random.randn(5)})
print(df)
group1 = df.groupby('key1')
for x in group1:
    print(x)
print(group1.size())
print(group1.sum())
print(group1.count())
print(group1['data1'].agg('mean'))
print(group1['data1'].agg('mean'))
print(group1.mean())
print(group1['data1','data2'].agg(['mean','sum']) )
print(df.groupby('key1').apply(np.mean) )
print(df.groupby(['key1','key2']).apply(np.mean))
print(df.groupby('key1').apply(np.mean))
print(group1['data1','data2'].agg(['mean','sum']).reset_index())
group2 = df.groupby(['key1','key2'])
for x in group2:
    print(x)

读写操作

data=pd.read_csv('stud7raw.csv',encoding='utf-8')
# data=pd.read_csv('stud7raw.csv',encoding='gbk')
print(data)
data.to_pickle('student.pickle')

常用函数

#计算最近15天均值
 df['avg'] = df['close'].rolling(window_size).apply(lambda x: sum(x) / len(x))

df['action'] = df[['close', 'avg']].apply(avg_buy, axis=1)#avg_buy为自定义的函数

da=[]
for i in range(len(chinaDayAddList)):
    da.append([chinaDayAddList[i]["date"],chinaDayAddList[i]["confirm"] , chinaDayAddList[i]["suspect"], chinaDayAddList[i]["dead"],chinaDayAddList[i]["heal"]])
print(da)
df = pd.DataFrame({})
df["date"] =  [x[0] for x in da]
df["confirm"] = [x[1] for x in da]
df["suspect"] =  [x[2] for x in da]
df["dead"] =  [x[3] for x in da]
df["heal"] =  [x[4] for x in da]
print(df)

#按照每学期给成绩评定等级
df.groupby(by=['学期'])['成绩'].agg({'平均值':numpy.mean,'最低分':numpy.min,'方差':numpy.var,'标准差':numpy.std})
bins =[min(df. 成绩)-1,60,80,max(df. 成绩)+1]
labels=['60及其以下','60到80','80及其以上']
成绩分层 = pandas.cut(df.成绩,bins,labels=labels)