数据处理之pandas的常用操作

1.导入库

import pandas as pd
import numpy as np

2.创建一个序列

s = pd.Series([1,3,6,np.nan,44,1]) #创建一个序列
s

dates  = pd.date_range('20201010',periods = 6) #创建时间序列
dates

3.创建一个dataframe

# 创建一个dataframe
df = pd.DataFrame(np.random.randn(6,4),index = dates ,columns= ['a','b','c','d'])
df

df1 = pd.DataFrame(np.arange(12).reshape((3,4)))
df1

# 用字典创建一个df
df2 = pd.DataFrame({'A':1.,
         'B':pd.Timestamp('20201010'),
         'C':pd.Series(1,index=list(range(4)),dtype='float32'),
         'D':np.array([3]*4,dtype = 'int32'),
         'E':pd.Categorical(['test','train','test','train']),
         'F':'foo'
})
df2

4.一些基本操作

# 查看数据类型
df2.dtypes

# 查看索引
df2.index

# 查看值
df2.values

# 查看描述型统计
df2.describe()

# 转置
df2.T

# 排序
df2.sort_index(axis=1,ascending=False) # 对列index倒序排列 正序为True

df2.sort_values(by = 'E')  # 对E的值进行排序

5.选择数据

dates = pd.date_range('20201010',periods= 6)
df = pd.DataFrame(np.arange(24).reshape((6,4)),index= dates,columns=['A','B','C','D'])
df

print(df['A']) #输出A列

df[0:3] # df的1到3行

df.loc['20201010'] # 选择指定行

df.loc[:,['A','C']] # 选取所有行以及A和C列

df.iloc[3:5,1:3] #通过iloc进行指定切片

df.iloc[[1,3,5],1:3]

# 是或否的筛选
df[df.A > 8]

6.赋值

# 赋值 
df.iloc[2,2] = 666666
df.loc['20201010','B'] = 999999
df.A[df.A > 4] = 0 # 把A列所有大于4的值变成0
df['F'] = np.nan # 新增一列F
df['E'] = pd.Series([1,2,3,4,5,6],index=pd.date_range('20201010',periods=6)) # 新增一列E 注意index要一样
df

7.处理丢失数据

dates = pd.date_range('20201010',periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)),index= dates,columns=['A','B','C','D'])
df.iloc[0,1] = np.nan
df.iloc[1,2] = np.nan
df

df.dropna(axis=0,how = 'any') # how = {'any','all'}  因为axis=0 any表示只要行中出现nan就drop掉，all表示全部是nan才drop掉

df.fillna(value= 0 ) # 将nan填充为0

np.any(df.isnull()) == True # 检查数据是否至少包含一个缺失值

8.pandas导入导出数据

# pandas 导入导出数据
data = pd.read_csv('student.csv')
data

# 存储
data.to_pickle('student.pickle')

9.合并concat

df1 = pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2,columns=['a','b','c','d'])

res = pd.concat([df1,df2,df3],axis=0,ignore_index = True) # 行合并
res

df1 = pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'],index = [1,2,3])
df2 = pd.DataFrame(np.ones((3,4))*1,columns=['b','c','d','e'],index = [2,3,4])
print(df1)
print(df2)

res = pd.concat([df1,df2],join='outer')  # 取并集合并
res 

res = pd.concat([df1,df2],join='inner',ignore_index = True)  # 取交集合并
res 

10.append合并

#append
df1 = pd.DataFrame(np.ones((3,4))*0,columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*1,columns=['a','b','c','d'],index = [2,3,4])
res = df1.append([df2,df3],ignore_index = True) # 用append合并表
res

s1 = pd.Series([1,2,3,4],index=['a','b','c','d']) 
res = df1.append(s1,ignore_index=True) #添加一行
res

#merge
left = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
                             'A': ['A0', 'A1', 'A2', 'A3'],
                             'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
                              'C': ['C0', 'C1', 'C2', 'C3'],
                              'D': ['D0', 'D1', 'D2', 'D3']})
print(left)
print(right)

res = pd.merge(left,right,on = 'key') # 通过‘key’列合并
res

left = pd.DataFrame({'key1': ['K0', 'K0', 'K1', 'K2'],
                             'key2': ['K0', 'K1', 'K0', 'K1'],
                             'A': ['A0', 'A1', 'A2', 'A3'],
                             'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key1': ['K0', 'K1', 'K1', 'K2'],
                              'key2': ['K0', 'K0', 'K0', 'K0'],
                              'C': ['C0', 'C1', 'C2', 'C3'],
                              'D': ['D0', 'D1', 'D2', 'D3']})

res = pd.merge(left,right,on = ['key1','key2']) # 通过‘key1’'key2'列合并
res

11.merge合并

# indicator
df1 = pd.DataFrame({'col1':[0,1], 'col_left':['a','b']})
df2 = pd.DataFrame({'col1':[1,2,2],'col_right':[2,2,2]})
print(df1)
print(df2)
res = pd.merge(df1, df2, on='col1', how='outer', indicator=True)
# give the indicator a custom name
res = pd.merge(df1, df2, on='col1', how='outer', indicator='indicator_column')
res

# merged by index
left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
                                  'B': ['B0', 'B1', 'B2']},
                                  index=['K0', 'K1', 'K2'])
right = pd.DataFrame({'C': ['C0', 'C2', 'C3'],
                                     'D': ['D0', 'D2', 'D3']},
                                      index=['K0', 'K2', 'K3'])
print(left)
print(right)
# left_index and right_index
res = pd.merge(left, right, left_index=True, right_index=True, how='outer')
#res = pd.merge(left, right, left_index=True, right_index=True, how='inner')
res

# handle overlapping 合并名字相同含义不同的数据
boys = pd.DataFrame({'k': ['K0', 'K1', 'K2'], 'age': [1, 2, 3]})
girls = pd.DataFrame({'k': ['K0', 'K0', 'K3'], 'age': [4, 5, 6]})
res = pd.merge(boys, girls, on='k', suffixes=['_boy', '_girl'], how='inner')
print(res)

12.pandas 画图

import matplotlib.pyplot as plt
data = pd.Series(np.random.randn(1000),index = np.arange(1000))
data = data.cumsum()
data

data.plot()
plt.show()

data = pd.DataFrame(np.random.randn(1000,4),
                   index = np.arange(1000),
                   columns=list('ABCD'))
data = data.cumsum()
data.plot()
plt.show()

ax = data.plot.scatter(x='A',y='B',color = 'DarkBlue',label = 'Class 1')
data.plot.scatter(x='A',y='C',color = 'DarkGreen',label = 'Class 2',ax =ax)
plt.show()