半小时拿下Python数据处理之Pandas篇

import pandas as pd

Pandas数据结构

Series

Series一维的数据结构

通过list构建Series

ser_obj =pd.Series(range(10,15))
print(type(ser_obj)) # <class 'pandas.core.series.Series'>
print(ser_obj)
<class 'pandas.core.series.Series'>
0    10
1    11
2    12
3    13
4    14
dtype: int32

获取数据

print(type(ser_obj.values)) # <class 'numpy.ndarray'>
print(ser_obj.values) # [10 11 12 13 14]
<class 'numpy.ndarray'>
[10 11 12 13 14]

获取索引

print(type(ser_obj.index)) # <class 'pandas.core.indexes.range.RangeIndex'>
print(ser_obj.index) # RangeIndex(start=0, stop=5, step=1)
<class 'pandas.core.indexes.range.RangeIndex'>
RangeIndex(start=0, stop=5, step=1)

注意索引对象不可变

# 索引对象不可变
ser_obj.index[0] = 2
---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-53-ce46badf9dd7> in <module>()
----> 1 ser_obj.index[0] = 2


G:\Anaconda3\lib\site-packages\pandas\core\indexes\base.py in __setitem__(self, key, value)
   1668
   1669     def __setitem__(self, key, value):
-> 1670         raise TypeError("Index does not support mutable operations")
   1671
   1672     def __getitem__(self, key):


TypeError: Index does not support mutable operations

预览数据

print(ser_obj.head(3))
0    10
1    11
2    12
dtype: int32

通过索引获取数据

print(ser_obj[0]) # 10
10

索引与数据的对应关系仍保持在数组运算的结果中

print(ser_obj > 12)
print(ser_obj[ser_obj > 12])
0    False
1    False
2    False
3     True
4     True
dtype: bool
3    13
4    14
dtype: int32

整合代码

# 通过list构建Series
ser_obj =pd.Series(range(10,15))
print(type(ser_obj)) # <class 'pandas.core.series.Series'>
print(ser_obj)

# 获取数据
print(type(ser_obj.values)) # <class 'numpy.ndarray'>
print(ser_obj.values) # [10 11 12 13 14]

# 获取索引
print(type(ser_obj.index)) # <class 'pandas.core.indexes.range.RangeIndex'>
print(ser_obj.index) # RangeIndex(start=0, stop=5, step=1)

# 预览数据
print(ser_obj.head(3))

#通过索引获取数据
print(ser_obj[0]) # 10

# 索引与数据的对应关系仍保持在数组运算的结果中
print(ser_obj > 12)
print(ser_obj[ser_obj > 12])
<class 'pandas.core.series.Series'>
0    10
1    11
2    12
3    13
4    14
dtype: int32
<class 'numpy.ndarray'>
[10 11 12 13 14]
<class 'pandas.core.indexes.range.RangeIndex'>
RangeIndex(start=0, stop=5, step=1)
0    10
1    11
2    12
dtype: int32
10
0    False
1    False
2    False
3     True
4     True
dtype: bool
3    13
4    14
dtype: int32

通过dict构建Series(注意:字典的key自动作为索引)

year_data = {2001: 17.8, 2002: 20.1, 2003: 16.5}
ser_obj2 = pd.Series(year_data)
print(type(ser_obj2)) # <class 'pandas.core.series.Series'>
print(ser_obj2)
<class 'pandas.core.series.Series'>
2001    17.8
2002    20.1
2003    16.5
dtype: float64

获取数据

print(type(ser_obj2.values)) # <class 'numpy.ndarray'>
print(ser_obj2.values) # [ 17.8  20.1  16.5]
<class 'numpy.ndarray'>
[ 17.8  20.1  16.5]

获取索引

print(type(ser_obj2.index)) # <class 'pandas.core.indexes.numeric.Int64Index'>
print(ser_obj2.index) # Int64Index([2001, 2002, 2003], dtype='int64')
<class 'pandas.core.indexes.numeric.Int64Index'>
Int64Index([2001, 2002, 2003], dtype='int64')

预览数据(head()不加参数则显示全部

print(ser_obj2.head())
2001    17.8
2002    20.1
2003    16.5
dtype: float64

通过索引获取数据

print(ser_obj2[2001]) # 17.8
17.8

整合代码

# 通过dict构建Series(注意:字典的key自动作为索引)
year_data = {2001: 17.8, 2002: 20.1, 2003: 16.5}
ser_obj2 = pd.Series(year_data)
print(type(ser_obj2)) # <class 'pandas.core.series.Series'>
print(ser_obj2)

# 获取数据
print(type(ser_obj2.values)) # <class 'numpy.ndarray'>
print(ser_obj2.values) # [ 17.8  20.1  16.5]

# 获取索引
print(type(ser_obj2.index)) # <class 'pandas.core.indexes.numeric.Int64Index'>
print(ser_obj2.index) # Int64Index([2001, 2002, 2003], dtype='int64')

# 预览数据(head()不加参数则显示全部)
print(ser_obj2.head())

#通过索引获取数据
print(ser_obj2[2001]) # 17.8
<class 'pandas.core.series.Series'>
2001    17.8
2002    20.1
2003    16.5
dtype: float64
<class 'numpy.ndarray'>
[ 17.8  20.1  16.5]
<class 'pandas.core.indexes.numeric.Int64Index'>
Int64Index([2001, 2002, 2003], dtype='int64')
2001    17.8
2002    20.1
2003    16.5
dtype: float64
17.8
DataFrame

一个Dataframe就是一张表格,Series表示的是一维数组,Dataframe则是一个二维数组,可以类比成一张excelspreadsheet也可以把 Dataframe当做一组Series的集合

通过ndarray构建DataFrame

import numpy as np

# 通过ndarray构建DataFrame
array = np.random.randn(5,4)
print(array)

df_obj = pd.DataFrame(array)
print(df_obj.head())
[[ 0.7346628  -1.13733651  0.72853785  0.38743511]
 [ 0.49549724  3.96998008  1.13567695 -0.21425912]
 [ 0.22094222  0.7766603   0.46086182  0.33199643]
 [-0.46279419  0.85898771  0.41993259 -0.61997791]
 [-0.83296535  1.19450707 -1.45531366 -0.13990243]]
          0         1         2         3
0  0.734663 -1.137337  0.728538  0.387435
1  0.495497  3.969980  1.135677 -0.214259
2  0.220942  0.776660  0.460862  0.331996
3 -0.462794  0.858988  0.419933 -0.619978
4 -0.832965  1.194507 -1.455314 -0.139902

通过dict构建DataFrame

dict_data = {'A': 1.,
             'B': pd.Timestamp('20180316'),
             'C': pd.Series(1, index=list(range(4)),dtype='float32'),
             'D': np.array([3] * 4,dtype='int32'),
             'E' : pd.Categorical(["Python","Java","C++","C#"])
            }
print(dict_data)
df_obj2 = pd.DataFrame(dict_data)
print(df_obj2.head())
{'A': 1.0, 'B': Timestamp('2018-03-16 00:00:00'), 'C': 0    1.0
1    1.0
2    1.0
3    1.0
dtype: float32, 'D': array([3, 3, 3, 3]), 'E': [Python, Java, C++, C#]
Categories (4, object): [C#, C++, Java, Python]}
     A          B    C  D       E
0  1.0 2018-03-16  1.0  3  Python
1  1.0 2018-03-16  1.0  3    Java
2  1.0 2018-03-16  1.0  3     C++
3  1.0 2018-03-16  1.0  3      C#

通过列索引获取列数据

print(df_obj2['A'])
print(type(df_obj2['A']))

print(df_obj2.A)
0    1.0
1    1.0
2    1.0
3    1.0
Name: A, dtype: float64
<class 'pandas.core.series.Series'>
0    1.0
1    1.0
2    1.0
3    1.0
Name: A, dtype: float64

通过行索引(.loc)获取行数据

print(df_obj2.loc[0])
print(type(df_obj2.loc[0]))
A                      1
B    2018-03-16 00:00:00
C                      1
D                      3
E                 Python
Name: 0, dtype: object
<class 'pandas.core.series.Series'>

增加列

df_obj2['F'] = df_obj2['D'] + 4
print(df_obj2.head())
     A          B    C  D       E  F
0  1.0 2018-03-16  1.0  3  Python  7
1  1.0 2018-03-16  1.0  3    Java  7
2  1.0 2018-03-16  1.0  3     C++  7
3  1.0 2018-03-16  1.0  3      C#  7

删除列

del(df_obj2['F'] )
print(df_obj2.head())
     A          B    C  D       E
0  1.0 2018-03-16  1.0  3  Python
1  1.0 2018-03-16  1.0  3    Java
2  1.0 2018-03-16  1.0  3     C++
3  1.0 2018-03-16  1.0  3      C#

整合代码

import numpy as np

# 通过ndarray构建DataFrame
array = np.random.randn(5,4)
print(array)

# 通过dict构建DataFrame
df_obj = pd.DataFrame(array)
print(df_obj.head())

dict_data = {'A': 1.,
             'B': pd.Timestamp('20180316'),
             'C': pd.Series(1, index=list(range(4)),dtype='float32'),
             'D': np.array([3] * 4,dtype='int32'),
             'E' : pd.Categorical(["Python","Java","C++","C#"])
            }
print(dict_data)
df_obj2 = pd.DataFrame(dict_data)
print(df_obj2.head())

# 通过列索引获取列数据
print(df_obj2['A'])
print(type(df_obj2['A']))

print(df_obj2.A)

# 通过行索引获取行数据
print(df_obj2.loc[0])
print(type(df_obj2.loc[0]))

# 增加列
df_obj2['G'] = df_obj2['D'] + 4
print(df_obj2.head())

# 删除列
del(df_obj2['G'] )
print(df_obj2.head())
[[ 0.23758715 -1.13751056 -0.0863061  -0.71309414]
 [ 0.08129935  1.32099551 -0.27057527  0.49270974]
 [ 0.96111551  1.08307556  1.5094844   0.96117055]
 [-0.31003598  1.33959047 -0.42150857 -1.20605423]
 [ 0.12655879 -1.01810288 -1.34025171  0.98758417]]
          0         1         2         3
0  0.237587 -1.137511 -0.086306 -0.713094
1  0.081299  1.320996 -0.270575  0.492710
2  0.961116  1.083076  1.509484  0.961171
3 -0.310036  1.339590 -0.421509 -1.206054
4  0.126559 -1.018103 -1.340252  0.987584
{'A': 1.0, 'B': Timestamp('2018-03-16 00:00:00'), 'C': 0    1.0
1    1.0
2    1.0
3    1.0
dtype: float32, 'D': array([3, 3, 3, 3]), 'E': [Python, Java, C++, C#]
Categories (4, object): [C#, C++, Java, Python]}
     A          B    C  D       E
0  1.0 2018-03-16  1.0  3  Python
1  1.0 2018-03-16  1.0  3    Java
2  1.0 2018-03-16  1.0  3     C++
3  1.0 2018-03-16  1.0  3      C#
0    1.0
1    1.0
2    1.0
3    1.0
Name: A, dtype: float64
<class 'pandas.core.series.Series'>
0    1.0
1    1.0
2    1.0
3    1.0
Name: A, dtype: float64
A                      1
B    2018-03-16 00:00:00
C                      1
D                      3
E                 Python
Name: 0, dtype: object
<class 'pandas.core.series.Series'>
     A          B    C  D       E  G
0  1.0 2018-03-16  1.0  3  Python  7
1  1.0 2018-03-16  1.0  3    Java  7
2  1.0 2018-03-16  1.0  3     C++  7
3  1.0 2018-03-16  1.0  3      C#  7
     A          B    C  D       E
0  1.0 2018-03-16  1.0  3  Python
1  1.0 2018-03-16  1.0  3    Java
2  1.0 2018-03-16  1.0  3     C++
3  1.0 2018-03-16  1.0  3      C#

Pandas 数据操作

import pandas as pd
Series索引
ser_obj = pd.Series(range(5), index = ['a', 'b', 'c', 'd', 'e'])
ser_obj.head()
a    0
b    1
c    2
d    3
e    4
dtype: int32

行索引

# 行索引
ser_obj['a'] #等同描述ser_obj[0]
0

切片索引可以按照默认索引号,也可以按照实际索引值

# 切片索引(按索引号)
ser_obj[1:3] #python索引默认是左闭右开
b    1
c    2
dtype: int32
# 切片索引(按索引值)
ser_obj['b':'d']
b    1
c    2
d    3
dtype: int32

不连续索引,同样可以按照默认索引号,也可以按照实际索引值

# 不连续索引表达一(按索引号)
ser_obj[[0, 2, 4]]
a    0
c    2
e    4
dtype: int32
# 不连续索引表达二(按索引值)
ser_obj[['a', 'e']]
a    0
e    4
dtype: int32

布尔索引

# 布尔索引
ser_bool = ser_obj > 2
print(ser_bool)
print()
print(ser_obj[ser_bool])
print()
print(ser_obj[ser_obj > 2])
a    False
b    False
c    False
d     True
e     True
dtype: bool

d    3
e    4
dtype: int32

d    3
e    4
dtype: int32
DataFrame索引
import numpy as np

df_obj = pd.DataFrame(np.random.randn(5,4), columns = ['a', 'b', 'c', 'd'])
df_obj.head()
abcd
00.9837901.0638040.854634-1.269025
10.161653-0.904602-1.8400410.138183
2-1.256608-1.740634-1.653686-0.412524
30.1657821.1160890.065008-1.693706
41.3139870.734437-0.625647-1.738446

列索引

# 列索引
print(type(df_obj['a'])) # 返回Series类型
df_obj['a'] # 返回对应列值
<class 'pandas.core.series.Series'>





0    0.983790
1    0.161653
2   -1.256608
3    0.165782
4    1.313987
Name: a, dtype: float64

行索引

# 行索引
print(type(df_obj.loc[0])) # 返回Series类型
df_obj.loc[0] # 返回对应行值
<class 'pandas.core.series.Series'>





a    0.983790
b    1.063804
c    0.854634
d   -1.269025
Name: 0, dtype: float64

不连续索引

#不连续列索引
df_obj[['a','c']]  #不连续列索引
ac
00.9837900.854634
10.161653-1.840041
2-1.256608-1.653686
30.1657820.065008
41.313987-0.625647
#不连续行索引
df_obj.loc[[1, 3]] #不连续行索引
abcd
10.161653-0.904602-1.8400410.138183
30.1657821.1160890.065008-1.693706

混合索引

# 混合索引 loc
print(df_obj.loc[0:2, 'a']) # 连续行加列索引(这里是从0-2)
print()
print(df_obj.loc[[0,2,4], 'a']) # 不连续行加列索引
0   -1.018941
1    0.089275
2   -2.210780
Name: a, dtype: float64

0   -1.018941
2   -2.210780
4    1.435787
Name: a, dtype: float64
运算与对齐
Series 对齐操作
s1 = pd.Series(range(10, 13), index = range(3))
s2 = pd.Series(range(20, 25), index = range(5))

print('s1: ' )
print(s1)

print('')

print('s2: ')
print(s2)
s1:
0    10
1    11
2    12
dtype: int32

s2:
0    20
1    21
2    22
3    23
4    24
dtype: int32
# Series 对齐运算
print(s1 + s2) # 没有对应上的部分会显示NaN
print()
print(s1.add(s2, fill_value = -1)) # 没有对应上的部分会填充-1,然后运算
print()
s3 = s1 + s2
s3_filled = s3.fillna(-1)
print(s3_filled) ## 先运算,然后NaN填充为-1
0    30.0
1    32.0
2    34.0
3     NaN
4     NaN
dtype: float64

0    30.0
1    32.0
2    34.0
3    22.0
4    23.0
dtype: float64

0    30.0
1    32.0
2    34.0
3    -1.0
4    -1.0
dtype: float64
DataFrame 对齐操作
import numpy as np

df1 = pd.DataFrame(np.ones((2,2)), columns = ['a', 'b'])
df2 = pd.DataFrame(np.ones((3,3)), columns = ['a', 'b', 'c'])

print('df1: ')
print(df1)

print('')
print('df2: ')
print(df2)
df1:
     a    b
0  1.0  1.0
1  1.0  1.0

df2:
     a    b    c
0  1.0  1.0  1.0
1  1.0  1.0  1.0
2  1.0  1.0  1.0
# DataFrame对齐操作
df1 + df2 # 没有对应上的部分会显示NaN
abc
02.02.0NaN
12.02.0NaN
2NaNNaNNaN
df1.add(df2, fill_value = 0) # 加法操作,没有对应上的补零
abc
02.02.01.0
12.02.01.0
21.01.01.0
df1 - df2 # 没有对应上的部分会显示NaN
abc
00.00.0NaN
10.00.0NaN
2NaNNaNNaN
df1.sub(df2, fill_value = 2) # 加法操作,没有对应上的补2(先补充后运算)
abc
00.00.01.0
10.00.01.0
21.01.01.0
df3 = df1 + df2
df3.fillna(100, inplace = True) # 先运行加法操作,没有对应上的补2(先运算,后补充)
df3
abc
02.02.0100.0
12.02.0100.0
2100.0100.0100.0
函数应用

可以与NumPy中的ufunc函数结合操作

# Numpy ufunc 函数
df = pd.DataFrame(np.random.randn(5,4) - 1)
df
0123
0-0.938212-2.487779-1.805374-1.130723
1-0.5334410.196536-1.094895-1.819312
2-3.2333180.255510-1.560183-2.404621
3-1.956924-2.947539-1.640760-0.757321
40.1986180.344484-0.893815-0.498036
np.abs(df) #取绝对值(还有其他诸多NumPy中的函数可以操作)
0123
00.9382122.4877791.8053741.130723
10.5334410.1965361.0948951.819312
23.2333180.2555101.5601832.404621
31.9569242.9475391.6407600.757321
40.1986180.3444840.8938150.498036

使用apply应用行或列数据

# 使用apply应用行或列数据
# f = lambda x : x.max() # lambda存在意义就是对简单函数的简洁表示
def f(x):
    return x.max()

df.apply(f) # 默认按行比较(得到每列的最大值)
0    0.198618
1    0.344484
2   -0.893815
3   -0.498036
dtype: float64
df.apply(lambda x : x.max(), axis=1) # 按列比较(得到每行的最大值)
0   -0.938212
1    0.196536
2    0.255510
3   -0.757321
4    0.344484
dtype: float64
df.apply(lambda x : x.max(), axis=0) # # 按行比较(得到每列的最大值)
0    0.198618
1    0.344484
2   -0.893815
3   -0.498036
dtype: float64

使用applymap应用到每个数据

# 使用applymap应用到每个数据
f2 = lambda x : '%.2f' % x #每个数据显示只保留两位小数
df.applymap(f2)
0123
0-0.94-2.49-1.81-1.13
1-0.530.20-1.09-1.82
2-3.230.26-1.56-2.40
3-1.96-2.95-1.64-0.76
40.200.34-0.89-0.50
排序

Series索引排序 & 值排序

#索引乱序生成
s4 = pd.Series([10,13,12,25,14], index = [2,1,5,3,4])
s4
2    10
1    13
5    12
3    25
4    14
dtype: int64
# 索引排序
s4.sort_index(ascending=False) #  索引倒序排列
5    12
4    14
3    25
2    10
1    13
dtype: int64
# 值排序
s4.sort_values()
2    10
5    12
1    13
4    14
3    25
dtype: int64

DataFrame 索引排序 & 值排序

df4 = pd.DataFrame(np.random.randn(3, 4),
                   index=[1,3,2],
                   columns=[1,4,2,3])
df4
1423
10.9481120.0763230.0896070.091737
3-1.2545561.4835040.4689950.286249
2-0.806738-0.842388-1.127489-0.020803
#按索引排序
df4.sort_index(ascending=False)# 对横轴按倒序排列
1423
3-1.2545561.4835040.4689950.286249
2-0.806738-0.842388-1.127489-0.020803
10.9481120.0763230.0896070.091737
#按索引排序
df4.sort_index(axis=1) #列轴按序排列
1234
10.9481120.0896070.0917370.076323
3-1.2545560.4689950.2862491.483504
2-0.806738-1.127489-0.020803-0.842388
#按列排序
df4.sort_values(by=1) # by参数的作用是针对某一(些)列进行排序(不能对行使用 by 参数)
1423
3-1.2545561.4835040.4689950.286249
2-0.806738-0.842388-1.127489-0.020803
10.9481120.0763230.0896070.091737
处理缺失数据

生成数据

df_data = pd.DataFrame([np.random.randn(3), [1., np.nan, np.nan],
                       [4., np.nan, np.nan], [1., np.nan, 2.]])
df_data.head()
012
01.089477-0.486706-0.322284
11.000000NaNNaN
24.000000NaNNaN
31.000000NaN2.000000

二值化(NaN为False,非NaN为True)

# isnull
df_data.isnull()
012
0FalseFalseFalse
1FalseTrueTrue
2FalseTrueTrue
3FalseTrueFalse

丢掉有NaN的行或列

# dropna
print(df_data.dropna()) #默认丢掉有NaN的行
print()
print(df_data.dropna(axis=1)) #丢掉有NaN的列
          0         1         2
0  1.089477 -0.486706 -0.322284

          0
0  1.089477
1  1.000000
2  4.000000
3  1.000000

填充NaN值

# fillna
df_data.fillna(-100.) # NaN值填充为-100
012
01.089477-0.486706-0.322284
11.000000-100.000000-100.000000
24.000000-100.000000-100.000000
31.000000-100.0000002.000000

数据统计计算和描述

常用的统计计算
df_obj = pd.DataFrame(np.random.randn(5,4), columns = ['a', 'b', 'c', 'd'])
df_obj
abcd
00.145119-2.3985950.6408060.696701
1-0.877139-0.261616-2.2117340.140729
2-0.6445450.523667-1.460002-0.341459
31.3692601.0399810.1640750.380755
40.089507-0.3710511.348191-0.828315
df_obj.sum()
a    0.082203
b   -1.467614
c   -1.518663
d    0.048410
dtype: float64
df_obj.max()
a    1.369260
b    1.039981
c    1.348191
d    0.696701
dtype: float64
df_obj.min(axis=1)
0   -2.398595
1   -2.211734
2   -1.460002
3    0.164075
4   -0.828315
dtype: float64
统计描述
df_obj.describe()
abcd
count5.0000005.0000005.0000005.000000
mean0.016441-0.293523-0.3037330.009682
std0.8785501.3119061.4846950.602578
min-0.877139-2.398595-2.211734-0.828315
25%-0.644545-0.371051-1.460002-0.341459
50%0.089507-0.2616160.1640750.140729
75%0.1451190.5236670.6408060.380755
max1.3692601.0399811.3481910.696701

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值