数据分析---3-1pandas的使用

目录

 数据索引、赋值

处理丢失数据、保存数据

合并contaneting

join拼接

append追加数据

merge合并相关操作

handle overlapping解决重合问题，进行标记

pandas plot数据可视化

---

基础知识

 数据索引、赋值

dates = pd.date_range('20190405', periods=6)
df = pd.DataFrame(np.arange(24).reshape(6,4), index=dates, columns=["A","B","C","D"])
print(df)
# df[0:3]按行索引,左闭右开
print(df[0:3], df['20190405':'20190406'])
# 选择行标签为20190406的这一行
print(df.loc['20190406 '])
# 选择列标签为A、B的俩列
print(df.loc['20190406',['A','B']])
print(df["A"], df.A)

# 根据位置序号选择：iloc
print(df.iloc[[3,4],[2,3]])
print(df.iloc[3:5,2:4])

# 还有一种混合筛选的方式，但是目前好像已经被弃用了

# Boolean indexing
print(df)
# 列标签A中>8的有(4,5,6)行
print(df.A > 8)
# 先选出了(4,5,6)行，故只输出4，5，6行
print(df[df.A > 8])

df['F']= np.nan
df.iloc[0:3,4] = [0,1,2]
# df['E']= pd.Series([1,2,3,4,5,6], index=pd.date_range('20190405', periods=6))
df['E'] = np.arange(6)
print(df)

处理丢失数据、保存数据

# 处理丢掉数据
print(df.dropna(axis=0, how="any"))
# 0表示第1维度(二维是行，三维是块),
# how默认为为any,表示该维度若至少存在1个nan，则删除该维度;all表示该维度全为nan，则删除该维度
print(df.fillna(value=666))
print(df.isnull())
print(np.any(df.isnull()) == True) # 如若dataframe中存在一个nan，即输出True

# #保存数据
# data = pd.read_csv("./filename.csv")
# print(data)
# data.to_csv("./new_filename.csv")

合并contaneting

# 合并contaneting
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d']) #全0
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d']) #全1
df3 = pd.DataFrame(np.ones((3,4))*2, columns=['a','b','c','d']) #全2
print(df1)
print(df2)
print(df3)
res = pd.concat([df1,df2,df3], axis=0, ignore_index=True)
# ignore_index=True忽略之前各个df的行序号，合并后默认从0开始排列
# 注意！axis=0，1，2 表示对1维(0)、二维(o第一维度行，1第二维度列）、三维处理（0第一维度块，1第二维度行，2第三维度列）

join拼接

#join拼接
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d']) #全0
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d','e']) #全1
print(df1)
print(df2)
# 默认方式为join="outer"（取二者并集，没有者用nan代替）;可选取inner方式（选取二者的交集）
res = pd.concat([df1,df2],join="inner",ignore_index=True)
print(res)

df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'], index =[1,2,3]) #全0
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d','e'], index = [2,3,4]) #全1
res = pd.concat([df1,df2],axis=1,join_axes=[df1.index])
# 以df1的索引为主索引进行合并，默认为将二者并起来，无数据部分用nan代替
print(res)

append追加数据

# append
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d']) #全0
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d']) #全1
df3 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d','e']) #全1
res1 = df1.append(df2, ignore_index=True)
print(res1)
# 没有数据的部分用nan代替，因为列序列不完全相同
res2 = df1.append([df2,df3], ignore_index=True)
print(res2)

# 给1个dataframe加1个Series
s1 = pd.Series([6,7,8,9], index=['a','b','c','d'])
res3 = df1.append(s1,ignore_index=True)
print(res3)

merge合并相关操作

# merge 2 dataframe by key/keys
left = pd.DataFrame({'key1':['K0','K1','K2','K3'],
                     'key2':['K0','K1','K2','K3'],
                     'A':['A0','A1','A1','A3'],
                     'B':['B0','B1','B2','B3']})
right = pd.DataFrame({'key1':['K0','K1','K2','K3'],
                     'key2':['K0','K1','K2','K3'],
                     'key3': ['K0','K1','K2','K3'],
                     'C':['C0','C1','C1','C3'],
                     'D':['D0','D1','D2','D3']})
print(left)
print(right)
res = pd.merge(left, right, on="key")
# 以key为主键进行合并

res1 = pd.merge(left, right, on=["key1","key2"])
# merge中how={"inner"\"outer"\"right"\"left"}
# 默认为inner合并（取交集）、concat默认为outer合并（取并集）

res = pd.merge(left, right, on=['key1', 'key2'], how='right')  # default for how='inner'
print(res)
# how = ['left', 'right', 'outer', 'inner']
# 此处操作均和数据库中的类似
# res = pd.merge(left, right, on=['key1', 'key2'], how='left')
# print(res)

# indicator
df1 = pd.DataFrame({'col1':[0,1], 'col_left':['a','b']})
df2 = pd.DataFrame({'col1':[1,2,2],'col_right':[2,2,2]})
print(df1)
print(df2)
res = pd.merge(df1, df2, on='col1', how='outer', indicator=True)
# indicator=True对两个dataframe合并方式进行显示
print(res)
# give the indicator a custom name
res = pd.merge(df1, df2, on='col1', how='outer', indicator='indicator_column')
print(res)

# merged by index
left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
                                  'B': ['B0', 'B1', 'B2']},
                                  index=['K0', 'K1', 'K2'])
right = pd.DataFrame({'C': ['C0', 'C2', 'C3'],
                                     'D': ['D0', 'D2', 'D3']},
                                      index=['K0', 'K2', 'K3'])
print(left)
print(right)
# left_index and right_index
# res = pd.merge(left, right, left_index=True, right_index=True, how='outer')
# print(res)
res = pd.merge(left, right, left_index=True, right_index=True, how='inner')
print(res)

handle overlapping解决重合问题，进行标记

# handle overlapping解决重合问题，进行标记
boys = pd.DataFrame({'num': ['K0', 'K1', 'K2'], 'age': [21, 22, 23]})
girls = pd.DataFrame({'num': ['K0', 'K0', 'K3'], 'age': [24, 25, 26]})
print(boys)
print(girls)
res1 = pd.merge(boys, girls, on='num', suffixes=['_boy', '_girl'], how='inner')
# suffixes对数据来源进行标识
res2 = pd.merge(boys, girls, on='num', suffixes=['_boy', '_girl'], how='outer')
print(res1)
print(res2)

pandas plot数据可视化

pandas plot数据可视化
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt



# Series
data = pd.Series(np.random.randn(1000), index=np.arange(1000))
# np.random.randn(1000)返回一组服从正态分布的样本值，其稚为1
# np.random.rand(1000)返回一组[0,1)均匀分布的随机样本值(注意左闭右开)
# 详细区别参见https://blog.csdn.net/zenghaitao0128/article/details/78556535

print(np.random.rand(1000))
print(data)
data = data.cumsum()
按照所给定的轴参数返回元素的梯形累计和，
axis=0，按照行累加。axis=1，按照列累加。
axis不给定具体值，就把numpy数组当成一个一维数组。
data.plot()

#
# DataFrame
data = pd.DataFrame(np.random.randn(1000, 4), index=np.arange(1000), columns=list("ABCD"))
#有两个及以上参数时，则返回对应维度的数组，能表示向量或矩阵
# 1000行4列的数组
data = data.cumsum()
# # plot methods:
# # 'bar', 'hist', 'box', 'kde', 'area', scatter', hexbin', 'pie'
ax = data.plot.scatter(x='A', y='B', color='DarkBlue', label="Class 1")
# 生成一个散点图ax对象
data.plot.scatter(x='A', y='C', color='LightGreen', label='Class 2', ax=ax)
# 散点图ax对象附加给参数ax，其中参数ax标识数轴==也就是两个图合称为一张图

plt.show()