pandas想一个字典形式的numpy,numpy是序列化好的
import pandas as pd
import numpy as np
s=pd.Series([1,3,6,np.nan,44,1])
print(s)
#创建一个dateform,创建日期当作行
dates=pd.date_range("20160101",periods=6)
print(dates)
#定义一个dataform,index表示行也可以用row,columns表示列
df=pd.DataFrame(np.random.randn(6,4),index=dates,columns=['a','b','c','d'])
print(df)
#不知道行和列,则默认有0,1,2,3
df1=pd.DataFrame(np.arange(12).reshape((3,4)))
print(df1)
#另外一种dataFrame的方法
df2=pd.DataFrame({
'A':1,
'B':pd.Timestamp('20130102'),
'C':pd.Series(1,index=list(range(4)),dtype='float32'),
'D':np.array([3]*4,dtype='int32'),
'E':pd.Categorical(['test','train','test','train']),
'F':'foo'
})
print(df2)
#返回类型
print(df2.dtypes)
#返回所有列的序号
print(df2.index)
#输出columns
print(df2.columns)
#打印所有的value
print(df2.values)
#打印他的描述
print(df2.describe())
#转置
print(df2.T)
#排序,axis=1表示按行,ascending=False表示按照倒叙
print(df2.sort_index(axis=1,ascending=False))
print(df2.sort_index(axis=0,ascending=False))
# 排序values
print(df2.sort_values(by='E'))
输出:
0 1.0
1 3.0
2 6.0
3 NaN
4 44.0
5 1.0
dtype: float64
DatetimeIndex([‘2016-01-01’, ‘2016-01-02’, ‘2016-01-03’, ‘2016-01-04’,
‘2016-01-05’, ‘2016-01-06’],
dtype=‘datetime64[ns]’, freq=‘D’)
a b c d
2016-01-01 -0.392326 -1.076249 2.059953 0.687583
2016-01-02 1.266574 1.562370 -1.000323 2.094809
2016-01-03 -0.010265 0.667070 1.964644 0.481243
2016-01-04 0.673467 0.336153 -1.310106 1.559293
2016-01-05 0.321785 0.806592 1.243590 0.075628
2016-01-06 0.522519 -0.734596 0.188656 -1.147041
0 1 2 3
0 0 1 2 3
1 4 5 6 7
2 8 9 10 11
A B C D E F
0 1 2013-01-02 1.0 3 test foo
1 1 2013-01-02 1.0 3 train foo
2 1 2013-01-02 1.0 3 test foo
3 1 2013-01-02 1.0 3 train foo
A int64
B datetime64[ns]
C float32
D int32
E category
F object
dtype: object
Int64Index([0, 1, 2, 3], dtype=‘int64’)
Index([‘A’, ‘B’, ‘C’, ‘D’, ‘E’, ‘F’], dtype=‘object’)
[[1 Timestamp(‘2013-01-02 00:00:00’) 1.0 3 ‘test’ ‘foo’]
[1 Timestamp(‘2013-01-02 00:00:00’) 1.0 3 ‘train’ ‘foo’]
[1 Timestamp(‘2013-01-02 00:00:00’) 1.0 3 ‘test’ ‘foo’]
[1 Timestamp(‘2013-01-02 00:00:00’) 1.0 3 ‘train’ ‘foo’]]
A C D
count 4.0 4.0 4.0
mean 1.0 1.0 3.0
std 0.0 0.0 0.0
min 1.0 1.0 3.0
25% 1.0 1.0 3.0
50% 1.0 1.0 3.0
75% 1.0 1.0 3.0
max 1.0 1.0 3.0
0 … 3
A 1 … 1
B 2013-01-02 00:00:00 … 2013-01-02 00:00:00
C 1 … 1
D 3 … 3
E test … train
F foo … foo
[6 rows x 4 columns]
F E D C B A
0 foo test 3 1.0 2013-01-02 1
1 foo train 3 1.0 2013-01-02 1
2 foo test 3 1.0 2013-01-02 1
3 foo train 3 1.0 2013-01-02 1
A B C D E F
3 1 2013-01-02 1.0 3 train foo
2 1 2013-01-02 1.0 3 test foo
1 1 2013-01-02 1.0 3 train foo
0 1 2013-01-02 1.0 3 test foo
A B C D E F
0 1 2013-01-02 1.0 3 test foo
2 1 2013-01-02 1.0 3 test foo
1 1 2013-01-02 1.0 3 train foo
3 1 2013-01-02 1.0 3 train foo
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
print(df)
输出:
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
2013-01-04 12 13 14 15
2013-01-05 16 17 18 19
2013-01-06 20 21 22 23
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
#选择打印某一列
print(df['A'],df.A)
输出:
2013-01-01 0
2013-01-02 4
2013-01-03 8
2013-01-04 12
2013-01-05 16
2013-01-06 20
Freq: D, Name: A, dtype: int64 2013-01-01 0
2013-01-02 4
2013-01-03 8
2013-01-04 12
2013-01-05 16
2013-01-06 20
Freq: D, Name: A, dtype: int64
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
#选择打印某一列
print(df['A'],df.A)
#选择某几行
print(df[0:3],'\n',df['2013-01-01':'2013-01-03'])
输出:
2013-01-01 0
2013-01-02 4
2013-01-03 8
2013-01-04 12
2013-01-05 16
2013-01-06 20
Freq: D, Name: A, dtype: int64 2013-01-01 0
2013-01-02 4
2013-01-03 8
2013-01-04 12
2013-01-05 16
2013-01-06 20
Freq: D, Name: A, dtype: int64
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
print(df)
# #选择打印某一列
# print(df['A'],df.A)
# #选择某几行
# print(df[0:3],'\n',df['2013-01-01':'2013-01-03'])
#select by label:loc
print(df.loc['20130102'])
输出:
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
2013-01-04 12 13 14 15
2013-01-05 16 17 18 19
2013-01-06 20 21 22 23
A 4
B 5
C 6
D 7
Name: 2013-01-02 00:00:00, dtype: int64
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
print(df)
# #选择打印某一列
# print(df['A'],df.A)
# #选择某几行
# print(df[0:3],'\n',df['2013-01-01':'2013-01-03'])
#select by label:loc
# print(df.loc['20130102'])
#打印所有行,某几列
print(df.loc[:,['A','B']])
#输出某一行,某几列
print(df.loc['20130102',['A','B']])
输出:
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
2013-01-04 12 13 14 15
2013-01-05 16 17 18 19
2013-01-06 20 21 22 23
A B
2013-01-01 0 1
2013-01-02 4 5
2013-01-03 8 9
2013-01-04 12 13
2013-01-05 16 17
2013-01-06 20 21
A 4
B 5
Name: 2013-01-02 00:00:00, dtype: int64
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
print(df)
# #选择打印某一列
# print(df['A'],df.A)
# #选择某几行
# print(df[0:3],'\n',df['2013-01-01':'2013-01-03'])
#select by label:loc
# print(df.loc['20130102'])
#打印所有行,某几列
# print(df.loc[:,['A','B']])
# #输出某一行,某几列
# print(df.loc['20130102',['A','B']])
#select by position:iloc
print(df.iloc[3:5,1:3])
#一行一行的筛选
print(df.iloc[[1,3,5],1:3])
#mixed selection:ix
print(df.ix[:3,['A','C']])
#Boolean indexing
print(df[df.A<8])
输出:
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
2013-01-04 12 13 14 15
2013-01-05 16 17 18 19
2013-01-06 20 21 22 23
B C
2013-01-04 13 14
2013-01-05 17 18
B C
2013-01-02 5 6
2013-01-04 13 14
2013-01-06 21 22
A C
2013-01-01 0 2
2013-01-02 4 6
2013-01-03 8 10
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
print(df)
df.iloc[2,2]=111
#定位修改值
df.iloc[2,2]=111
df.loc['20130101','B']=222
#当A>4的时候,大于4的所有行都变为0
df[df.A>4]=0
#若只想改变某一列
df.A[df.A>4]=0
#若想增加一列
df['F']=np.nan
#增加的一列是序列化的
df['E']=pd.Series([1,2,3,4,5,6],index=pd.date_range('20130101',periods=6))
print(df)
输出:
A B C D
2013-01-01 0 1 2 3
2013-01-02 4 5 6 7
2013-01-03 8 9 10 11
2013-01-04 12 13 14 15
2013-01-05 16 17 18 19
2013-01-06 20 21 22 23
A B C D
2013-01-01 0 222 2 3
2013-01-02 4 5 6 7
2013-01-03 0 0 0 0
2013-01-04 0 0 0 0
2013-01-05 0 0 0 0
2013-01-06 0 0 0 0
只改变某一列
A B C D
2013-01-01 0 222 2 3
2013-01-02 4 5 6 7
2013-01-03 0 9 111 11
2013-01-04 0 13 14 15
2013-01-05 0 17 18 19
2013-01-06 0 21 22 23
增加一列:
A B C D F
2013-01-01 0 222 2 3 NaN
2013-01-02 4 5 6 7 NaN
2013-01-03 0 9 111 11 NaN
2013-01-04 0 13 14 15 NaN
2013-01-05 0 17 18 19 NaN
2013-01-06 0 21 22 23 NaN
A B C D F E
2013-01-01 0 222 2 3 NaN 1
2013-01-02 4 5 6 7 NaN 2
2013-01-03 0 9 111 11 NaN 3
2013-01-04 0 13 14 15 NaN 4
2013-01-05 0 17 18 19 NaN 5
2013-01-06 0 21 22 23 NaN 6
如何处理丢失数据
import pandas as pd
import numpy as np
dates=pd.date_range('20130101',periods=6)
df=pd.DataFrame(np.arange(24).reshape((6,4)),index=dates,columns=['A','B','C','D'])
df.iloc[0,1]=np.nan
df.iloc[1,2]=np.nan
print(df)
#按行丢掉,axis=0表示按照行,how='any'表示有任何的nan,how={'any','all'}
print(df.dropna(axis=0,how='any'))
#将nan进行填充数字0
print(df.fillna(value=0))
#如果有值返回false,是nan则返回true
print(df.isnull())
#如果数据特别多则检查是否有丢失数据
print(np.any(df.isnull())==True)
输出:
A B C D
2013-01-01 0 NaN 2.0 3
2013-01-02 4 5.0 NaN 7
2013-01-03 8 9.0 10.0 11
2013-01-04 12 13.0 14.0 15
2013-01-05 16 17.0 18.0 19
2013-01-06 20 21.0 22.0 23
A B C D
2013-01-03 8 9.0 10.0 11
2013-01-04 12 13.0 14.0 15
2013-01-05 16 17.0 18.0 19
2013-01-06 20 21.0 22.0 23
A B C D
2013-01-01 0 0.0 2.0 3
2013-01-02 4 5.0 0.0 7
2013-01-03 8 9.0 10.0 11
2013-01-04 12 13.0 14.0 15
2013-01-05 16 17.0 18.0 19
2013-01-06 20 21.0 22.0 23
A B C D
2013-01-01 False True False False
2013-01-02 False False True False
2013-01-03 False False False False
2013-01-04 False False False False
2013-01-05 False False False False
2013-01-06 False False False False
True
pandas如何导入导出数据
read_csv
read_excel
read_hdf
read_json
read_msgpack
read_html
read_gbq
read_stata
read_sas
read_clipboard
read_pickle
保存
to_csv
…与上面对应
import pandas as pd
data=pd.read_csv('student.csv')
print(data)
输出,自动加上索引
Student ID name age gender
0 1100 Kelly 22 Female
1 1101 Clo 21 Male
2 1102 Bob 23 Male
3 1103 Cindy 20 Female
4 1104 Killy 21 Male
5 1105 Micheal 22 Female
6 1106 Jack 23 Female
7 1107 John 22 Female
8 1108 Luna 20 Male
9 1109 Lisa 23 Female
10 1110 Alice 22 Male
11 1111 David 21 Male
12 1112 H 21 Female
13 1113 dwe 23 Female
14 1114 wd 22 Female
pandas合并
import pandas as pd
import numpy as np
#concatenating
df1=pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'])
df2=pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'])
df3=pd.DataFrame(np.ones((3,4))*2,columns=['a','b','c','d'])
print(df1)
print(df2)
print(df3)
#axis=0,按竖向合并,gnore_index=True,避免前面的索引呈现012012012
res=pd.concat([df1,df2,df3],axis=0,ignore_index=True)
print(res)
输出:
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
a b c d
0 1.0 1.0 1.0 1.0
1 1.0 1.0 1.0 1.0
2 1.0 1.0 1.0 1.0
a b c d
0 2.0 2.0 2.0 2.0
1 2.0 2.0 2.0 2.0
2 2.0 2.0 2.0 2.0
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 1.0 1.0 1.0 1.0
4 1.0 1.0 1.0 1.0
5 1.0 1.0 1.0 1.0
6 2.0 2.0 2.0 2.0
7 2.0 2.0 2.0 2.0
8 2.0 2.0 2.0 2.0
import pandas as pd
import numpy as np
#concatenating
df1=pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'],index=[1,2,3])
df2=pd.DataFrame(np.ones((3,4))*1,columns=['b','c','d','e'],index=[2,3,4])
print(df1)
print(df2)
#默认join是outer
res=pd.concat([df1,df2],join='outer',sort=True)
print(res)
#裁剪掉有NaN的项
res2=pd.concat([df1,df2],join='inner',sort=True,ignore_index=True)
print(res2)
#Join_axies,表示按照谁的索引来添加,因为二者的索引不同,没有的则为NaN
res3=pd.concat([df1,df2],axis=1,join_axes=[df1.index])
print(res3)
输出:
a b c d
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 0.0 0.0 0.0 0.0
b c d e
2 1.0 1.0 1.0 1.0
3 1.0 1.0 1.0 1.0
4 1.0 1.0 1.0 1.0
a b c d e
1 0.0 0.0 0.0 0.0 NaN
2 0.0 0.0 0.0 0.0 NaN
3 0.0 0.0 0.0 0.0 NaN
2 NaN 1.0 1.0 1.0 1.0
3 NaN 1.0 1.0 1.0 1.0
4 NaN 1.0 1.0 1.0 1.0
b c d
1 0.0 0.0 0.0
2 0.0 0.0 0.0
3 0.0 0.0 0.0
2 1.0 1.0 1.0
3 1.0 1.0 1.0
4 1.0 1.0 1.0
a b c d b c d e
1 0.0 0.0 0.0 0.0 NaN NaN NaN NaN
2 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
3 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
import pandas as pd
import numpy as np
#append,添加列
df1=pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'])
df2=pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'])
res=df1.append(df2,ignore_index=True)
print(res)
#也可以添加一行
s1=pd.Series([1,2,3,4],index=['a','b','c','d'])
res=df1.append(s1,ignore_index=True)
print(res)
输出:
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 1.0 1.0 1.0 1.0
4 1.0 1.0 1.0 1.0
5 1.0 1.0 1.0 1.0
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 1.0 2.0 3.0 4.0
#merging two df by key/keys.(may be used in database)
#simple example
import numpy as np
import pandas as pd
left=pd.DataFrame({'key':['K0','K1','K2','K3'],
'A':['A0','A1','A2','A3'],
'B':['B0','B1','B2','B3']})
right=pd.DataFrame({'key':['K0','K1','K2','K3'],
'C':['C0','C1','C2','C3'],
'D':['D0','D1','D2','D3']})
print(left)
print(right)
#基于key的column进行合并
res=pd.merge(left,right,on='key')
print(res)
输出:
key A B
0 K0 A0 B0
1 K1 A1 B1
2 K2 A2 B2
3 K3 A3 B3
key C D
0 K0 C0 D0
1 K1 C1 D1
2 K2 C2 D2
3 K3 C3 D3
key A B C D
0 K0 A0 B0 C0 D0
1 K1 A1 B1 C1 D1
2 K2 A2 B2 C2 D2
3 K3 A3 B3 C3 D3
import numpy as np
import pandas as pd
left=pd.DataFrame({'key1':['K0','K0','K1','K2'],
'key2':['K0','K1','K0','K1'],
'A':['A0','A1','A2','A3'],
'B':['B0','B1','B2','B3']})
right=pd.DataFrame({'key1':['K0','K1','K1','K2'],
'key2':['K0','K0','K0','K0'],
'C':['C0','C1','C2','C3'],
'D':['D0','D1','D2','D3']})
print(left)
print(right)
#默认的方式是inner,how=['left','right','outer','inner']
res=pd.merge(left,right,on=['key1','key2'],how='inner')
print(res)
#不管有没有都加上
res=pd.merge(left,right,on=['key1','key2'],how='outer')
print(res)
res=pd.merge(left,right,on=['key1','key2'],how='right')
print(res)
输出:
key1 key2 A B
0 K0 K0 A0 B0
1 K0 K1 A1 B1
2 K1 K0 A2 B2
3 K2 K1 A3 B3
key1 key2 C D
0 K0 K0 C0 D0
1 K1 K0 C1 D1
2 K1 K0 C2 D2
3 K2 K0 C3 D3
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K1 K0 A2 B2 C1 D1
2 K1 K0 A2 B2 C2 D2
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K0 K1 A1 B1 NaN NaN
2 K1 K0 A2 B2 C1 D1
3 K1 K0 A2 B2 C2 D2
4 K2 K1 A3 B3 NaN NaN
5 K2 K0 NaN NaN C3 D3
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K1 K0 A2 B2 C1 D1
2 K1 K0 A2 B2 C2 D2
3 K2 K0 NaN NaN C3 D3
#indicator
import numpy as np
import pandas as pd
df1=pd.DataFrame({'col1':[0,1],'col_left':['a','b']})
df2=pd.DataFrame({'col1':[1,2,2],'col_right':[2,2,2]})
print(df1)
print(df2)
#默认indicator是false
res=pd.merge(df1,df2,on='col1',how='outer',indicator=True)
#give the indicator a custom name
res=pd.merge(df1,df2,on='col1',how='outer',indicator='indicator_column')
print(res)
输出:
col1 col_left
0 0 a
1 1 b
col1 col_right
0 1 2
1 2 2
2 2 2
col1 col_left col_right _merge
0 0 a NaN left_only
1 1 b 2.0 both
2 2 NaN 2.0 right_only
3 2 NaN 2.0 right_only
col1 col_left col_right indicator_column
0 0 a NaN left_only
1 1 b 2.0 both
2 2 NaN 2.0 right_only
3 2 NaN 2.0 right_only
#merged by index
import numpy as np
import pandas as pd
left=pd.DataFrame({
'A':['A0','A1','A2'],
'B':['B0','B1','B2']
},index=['K0','K1','K2'])
right=pd.DataFrame({
'C':['C0','C2','C3'],
'D':['D0','D1','D3']
},index=['K0','K2','K3'])
print(left)
print(right)
#left_index and right_index
res=pd.merge(left,right,left_index=True,right_index=True,how='outer')
print(res)
输出:
A B
K0 A0 B0
K1 A1 B1
K2 A2 B2
C D
K0 C0 D0
K2 C2 D1
K3 C3 D3
A B C D
K0 A0 B0 C0 D0
K1 A1 B1 NaN NaN
K2 A2 B2 C2 D1
K3 NaN NaN C3 D3
#merging two df by key/keys.(may be used in database)
#handle overlapping
import numpy as np
import pandas as pd
boys=pd.DataFrame({
'k':['K0','K1','K2'],'age':[1,2,3]
})
girls=pd.DataFrame({'k':['K0','K0','k3'],'age':[4,5,6]})
print(boys)
print(girls)
#使用下标区分
res=pd.merge(boys,girls,on='k',suffixes=['_boy','_girl'],how='outer')
print(res)
输出:
k age
0 K0 1
1 K1 2
2 K2 3
k age
0 K0 4
1 K0 5
2 k3 6
k age_boy age_girl
0 K0 1.0 4.0
1 K0 1.0 5.0
2 K1 2.0 NaN
3 K2 3.0 NaN
4 k3 NaN 6.0
plot画图
#plot图表
import pandas as pd
import numpy as np
#数据可视化必备
import matplotlib.pyplot as plt
#plot data
#Series
# data=pd.Series(np.random.randn(1000),index=np.arange(1000))
# data=data.cumsum()
# data.plot()
# plt.show()
#DataFrame
data=pd.DataFrame(np.random.randn(1000,4),
index=np.arange(1000),
columns=list('ABCD'))
data=data.cumsum()
#默认输出前5行
# print(data.head(3))
# data.plot()
# plt.show()
#plot methods:
#bar,hist,box,kde,area,scatter,hexbin,pie
ax=data.plot.scatter(x='A',y='B',color='DarkBlue',label='Class 1')
data.plot.scatter(x='A',y='C',color='DarkGreen',label='Class 2',ax=ax)
plt.show()