2.1. pandas数据结构介绍

2.1 pandas数据结构介绍

import pandas as pd

from pandas import Series, DataFrame

import numpy as np

2.1.1 Series（数据+数据标签（即索引））

Series是⼀种类似于⼀维数组的对象(有点像dictionary)，它由⼀组数据（各种NumPy数据类型）以及⼀组与之相关的**数据标签（即索引）**组成。

obj = pd.Series([4, 7, -5, 3])

obj

0    4
1    7
2   -5
3    3
dtype: int64

索引在左边，默认是从0到N-1，值在右边

2.1.1.1 通过value可以获得值

# values
obj.values

array([ 4,  7, -5,  3], dtype=int64)

2.1.1.2 通过index可以获得索引

# index
obj.index

RangeIndex(start=0, stop=4, step=1)

2.1.1.3 自定义索引

obj2 = pd.Series([4,7,-5,3], index=['d','c','b','a'])

obj2

d    4
c    7
b   -5
a    3
dtype: int64

obj2.index

Index(['d', 'c', 'b', 'a'], dtype='object')

2.1.1.4 修改索引

obj

0    4
1    7
2   -5
3    3
dtype: int64

obj.index = ['Alice', 'Bob', 'Eve', 'Steve']

obj

Alice    4
Bob      7
Eve     -5
Steve    3
dtype: int64

2.1.1.5 通过索引的方式找值

obj2['a']

3

obj2[['a','b']] #找多组值里面要传index的list

a    3
b   -5
dtype: int64

2.1.1.6 进行NumPy运算时候

obj2[obj2 >3]

d    4
c    7
dtype: int64

obj2*2

d     8
c    14
b   -10
a     6
dtype: int64

np.exp(obj2)

d      54.598150
c    1096.633158
b       0.006738
a      20.085537
dtype: float64

2.1.1.7 判断index是否在Series内

'b' in obj2

True

'e' in obj2

False

2.1.1.8 通过字典创建Series

dict_data = {'Ohio': 35000, 'Texas': 71000, 'Oregon': 16000, 'Utah': 5000}

obj3 = pd.Series(dict_data)

obj3

Ohio      35000
Texas     71000
Oregon    16000
Utah       5000
dtype: int64

# 传入排好序的字典的键以改变顺序
states = ['California','Ohio','Oregon','Texas']

obj4 = pd.Series(dict_data,states)

obj4 # Utah不在states中，所以它被删除了

California        NaN
Ohio          35000.0
Oregon        16000.0
Texas         71000.0
dtype: float64

2.1.1.9 检测缺失数据

pd.isnull(obj4)

California     True
Ohio          False
Oregon        False
Texas         False
dtype: bool

pd.notnull(obj4)

California    False
Ohio           True
Oregon         True
Texas          True
dtype: bool

2.1.1.10 数据对齐

• 根据运算的索引标签自动对齐数据
• 类似数据库的join操作

obj4.isnull()

California     True
Ohio          False
Oregon        False
Texas         False
dtype: bool

obj4.notnull()

California    False
Ohio           True
Oregon         True
Texas          True
dtype: bool

obj3+obj4

obj3+obj4

California         NaN
Ohio           70000.0
Oregon         32000.0
Texas         142000.0
Utah               NaN
dtype: float64

2.1.1.10 name属性

• 取别名
• 设置了name之后在打印的时候会显现

obj4.name = 'population'

obj4.index.name = 'state'

obj4

state
California        NaN
Ohio          35000.0
Oregon        16000.0
Texas         71000.0
Name: population, dtype: float64

2.1.2 DataFrame

• 表格型数据结构
  • 一组有序的列，每列可以是不同的值类型
  • 有行索引和列索引
• DataFrame内部的数据是以一个或多个二维块存放的

2.1.2.1 创建

• 通过字典创建
• 通过list创建，然后再确定index和columns的序号
• 通过嵌套字典
• 通过pd.Series组成的字典

# 通过字典创建
data = {'state': ['Ohio','Ohio','Ohio','Nevada','Nevada','Nevada'],
        'year': [2000, 2001, 2002, 2003, 2004, 2005],
        'pop': [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]}

frame = pd.DataFrame(data)

frame

	state	year	pop
0	Ohio	2000	1.5
1	Ohio	2001	1.7
2	Ohio	2002	3.6
3	Nevada	2003	2.4
4	Nevada	2004	2.9
5	Nevada	2005	3.2

data2 = [['Ohio','Ohio','Ohio','Nevada','Nevada','Nevada'],
         [2000, 2001, 2002, 2003, 2004, 2005],
         [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]]

# 通过list创建，然后再确定index和columns的序号
pd.DataFrame(data2,index=['state','year','pop'],columns=[1,2,3,4,5,6])

	1	2	3	4	5	6
state	Ohio	Ohio	Ohio	Nevada	Nevada	Nevada
year	2000	2001	2002	2003	2004	2005
pop	1.5	1.7	3.6	2.4	2.9	3.2

# 通过嵌套字典
data3 = {'Nevada': {2001: 2.4, 2002: 2.9},
         'Ohio': {2000:1.5, 2001: 1.7, 2002: 3.6}
         }

frame3 = pd.DataFrame(data3)

frame3

	Nevada	Ohio
2001	2.4	1.7
2002	2.9	3.6
2000	NaN	1.5

# 通过pd.Series组成的字典
pdata = {'Ohio':frame3['Ohio'][:-1],
         'Nevada':frame3['Nevada'][:2]}

pd.DataFrame(pdata)

	Ohio	Nevada
2001	1.7	2.4
2002	3.6	2.9

2.1.2.2 head()/tail()

• 对于比较庞大的数据，head()会选取0~4行（共5行数据）
• tail()会选取倒数5行

frame.head()

	state	year	pop
0	Ohio	2000	1.5
1	Ohio	2001	1.7
2	Ohio	2002	3.6
3	Nevada	2003	2.4
4	Nevada	2004	2.9

frame.tail()

	state	year	pop
1	Ohio	2001	1.7
2	Ohio	2002	3.6
3	Nevada	2003	2.4
4	Nevada	2004	2.9
5	Nevada	2005	3.2

2.1.2.3 指定列序列

• 按自己想要的顺序指定

# 指定列序列
pd.DataFrame(data, columns=['year','state','pop'])

	year	state	pop
0	2000	Ohio	1.5
1	2001	Ohio	1.7
2	2002	Ohio	3.6
3	2003	Nevada	2.4
4	2004	Nevada	2.9
5	2005	Nevada	3.2

2.1.2.4 扩增列无数据会用NaN填充

frame2 = pd.DataFrame(data,columns=['year','state','pop','debt'],index=['one','two','three','four','five','six'])

frame2

	year	state	pop	debt
one	2000	Ohio	1.5	NaN
two	2001	Ohio	1.7	NaN
three	2002	Ohio	3.6	NaN
four	2003	Nevada	2.4	NaN
five	2004	Nevada	2.9	NaN
six	2005	Nevada	3.2	NaN

2.1.2.5 获取某行/列

#属性的方法
frame2.state

one        Ohio
two        Ohio
three      Ohio
four     Nevada
five     Nevada
six      Nevada
Name: state, dtype: object

#字典的方法
frame2['state']

one        Ohio
two        Ohio
three      Ohio
four     Nevada
five     Nevada
six      Nevada
Name: state, dtype: object

#获取某行
frame2.loc['one'] 

year     2000
state    Ohio
pop       1.5
debt     -1.2
Name: one, dtype: object

2.1.2.6 赋值

#通过列表
frame2['debt']=np.arange(1,7) 

frame2

	year	state	pop	debt
one	2000	Ohio	1.5	1
two	2001	Ohio	1.7	2
three	2002	Ohio	3.6	3
four	2003	Nevada	2.4	4
five	2004	Nevada	2.9	5
six	2005	Nevada	3.2	6

#通过列表
frame2['debt']=[4,4,4,4,4,4] 

frame2

	year	state	pop	debt
one	2000	Ohio	1.5	4
two	2001	Ohio	1.7	4
three	2002	Ohio	3.6	4
four	2003	Nevada	2.4	4
five	2004	Nevada	2.9	4
six	2005	Nevada	3.2	4

#赋相同的值也可以直接用标量
frame2['debt']=3 

frame2

	year	state	pop	debt
one	2000	Ohio	1.5	3
two	2001	Ohio	1.7	3
three	2002	Ohio	3.6	3
four	2003	Nevada	2.4	3
five	2004	Nevada	2.9	3
six	2005	Nevada	3.2	3

#通过pd.Series()赋值
val = pd.Series([-1.2,-1.5,-1.7], index = ['one','three','six'])

val

one     -1.2
three   -1.5
six     -1.7
dtype: float64

frame2['debt']=val

frame2

	year	state	pop	debt
one	2000	Ohio	1.5	-1.2
two	2001	Ohio	1.7	NaN
three	2002	Ohio	3.6	-1.5
four	2003	Nevada	2.4	NaN
five	2004	Nevada	2.9	NaN
six	2005	Nevada	3.2	-1.7

2.1.2.7 新建某行/列

• 列：frame[column]；pd.DataFrame(frame, columns = [])
• 行：frame.loc[index]；pd.DataFrame(frame, index = [])

frame2['estern'] = frame2.state =='Ohio' #列

frame2

	year	state	pop	debt	estern
one	2000	Ohio	1.5	-1.2	True
two	2001	Ohio	1.7	NaN	True
three	2002	Ohio	3.6	-1.5	True
four	2003	Nevada	2.4	NaN	False
five	2004	Nevada	2.9	NaN	False
six	2005	Nevada	3.2	-1.7	False

frame2.loc['seven'] = [2005, 'Nevada', '3.0', '-1.7', False] #行

frame2

	year	state	pop	debt	estern
one	2000	Ohio	1.5	-1.2	True
two	2001	Ohio	1.7	NaN	True
three	2002	Ohio	3.6	-1.5	True
four	2003	Nevada	2.4	NaN	False
five	2004	Nevada	2.9	NaN	False
six	2005	Nevada	3.2	-1.7	False
seven	2005	Nevada	3.0	-1.7	False

frame3

	Nevada	Ohio
2001	2.4	1.7
2002	2.9	3.6
2000	NaN	1.5

pd.DataFrame(frame3, index=[2001,2002,2003]) #列，且指定了列顺序

	Nevada	Ohio
2001	2.4	1.7
2002	2.9	3.6
2003	NaN	NaN

2.1.2.8 删除行/列

• drop()可以删除行/列，默认axis=0（行）,不是实际删除
• del函数可以删除列，是实际删除

frame2.drop('six') #删除行，只是在视图中删除（不是实际删除）

	year	state	pop	debt	estern
one	2000	Ohio	1.5	-1.2	True
two	2001	Ohio	1.7	NaN	True
three	2002	Ohio	3.6	-1.5	True
four	2003	Nevada	2.4	NaN	False
five	2004	Nevada	2.9	NaN	False
seven	2005	Nevada	3.0	-1.7	False

frame2.drop('estern',axis=1)

	year	state	pop	debt
one	2000	Ohio	1.5	-1.2
two	2001	Ohio	1.7	NaN
three	2002	Ohio	3.6	-1.5
four	2003	Nevada	2.4	NaN
five	2004	Nevada	2.9	NaN
six	2005	Nevada	3.2	-1.7
seven	2005	Nevada	3.0	-1.7

del frame2['estern']

frame2

	year	state	pop	debt
one	2000	Ohio	1.5	-1.2
two	2001	Ohio	1.7	NaN
three	2002	Ohio	3.6	-1.5
four	2003	Nevada	2.4	NaN
five	2004	Nevada	2.9	NaN
six	2005	Nevada	3.2	-1.7
seven	2005	Nevada	3.0	-1.7

2.1.2.9 转置

frame3

	Nevada	Ohio
2001	2.4	1.7
2002	2.9	3.6
2000	NaN	1.5

frame3.T

	2001	2002	2000
Nevada	2.4	2.9	NaN
Ohio	1.7	3.6	1.5

pd.DataFrame(frame3, index = [2001,2002,2003])

	Nevada	Ohio
2001	2.4	1.7
2002	2.9	3.6
2003	NaN	NaN

2.1.2.10 获取行，列，值

frame3.index #返回行

Int64Index([2001, 2002, 2000], dtype='int64')

frame3.columns #返回列

Index(['Nevada', 'Ohio'], dtype='object')

frame3.values #返回的是ndarray

array([[2.4, 1.7],
       [2.9, 3.6],
       [nan, 1.5]])

frame2.values #value的dtype会选择能兼容全部值的

array([[2000, 'Ohio', 1.5, -1.2],
       [2001, 'Ohio', 1.7, nan],
       [2002, 'Ohio', 3.6, -1.5],
       [2003, 'Nevada', 2.4, nan],
       [2004, 'Nevada', 2.9, nan],
       [2005, 'Nevada', 3.2, -1.7],
       [2005, 'Nevada', '3.0', '-1.7']], dtype=object)

2.1.3 索引对象 (Index)

1. 类似数组
2. 类似集合，但可以包含重复元素

obj = pd.Series(range(3),index=['a', 'b', 'c'])

obj.index #Index(['a', 'b', 'c'], dtype='object')

Index(['a', 'b', 'c'], dtype='object')

obj.index[1:] #Index(['b', 'c'], dtype='object')

Index(['b', 'c'], dtype='object')

2.1.3.1 index不可变

obj.index[0]='d' # 报错：TypeError: Index does not support mutable operations

---------------------------------------------------------------------------

TypeError                                 Traceback (most recent call last)

<ipython-input-142-d2bd8d3dd4c5> in <module>
----> 1 obj.index[0]='d'


F:\Anaconda3\lib\site-packages\pandas\core\indexes\base.py in __setitem__(self, key, value)
   3908 
   3909     def __setitem__(self, key, value):
-> 3910         raise TypeError("Index does not support mutable operations")
   3911 
   3912     def __getitem__(self, key):


TypeError: Index does not support mutable operations

2.1.3.2 index可共享

labels = pd.Index(np.arange(3)) 

labels # Int64Index([0, 1, 2], dtype='int64')

Int64Index([0, 1, 2], dtype='int64')

obj2 = pd.Series(['a','b','c'],index=labels)

obj2

0    a
1    b
2    c
dtype: object

obj2.index is labels # 判断两者是否相同

True

2.1.3.3 Index可包含重复标签

duplicates = pd.Index(['abc','abc','lalala','Jeff'])

duplicates

Index(['abc', 'abc', 'lalala', 'Jeff'], dtype='object')

2.1.3.4 Index的方法和属性

方法	说明
append	把当前Index对象和另一个Index连接起来
difference	计算两个集合的差集
intersection	计算两个集合的交集
union	计算两个集合的并集,注意：union和append得到的结果不一定一样
isin	返回的是bool类型的对应索引处元素是否在另一个Index内
delete	删除索引处的元素，并返回新的Index，旧Index索引处元素并未被删除
drop	删除值,并返回新的Index（只有当值是unique的时候可以使用）
insert	将元素插入索引处，会覆盖原有值，并得到新的Index
is_monotonic	当每个元素全都大于前一个元素的时候，返回True
is_unique	Index内没有重复元素就返回True
unique	返回没有重复的原Index中的所有元素组成的新Index，比如[1,2,3,4,1,2,3]->[1,2,3,4]

下面是按顺序的测试：

data1 = pd.Index([1,2,3,4,5])

data2 = pd.Index([4,5,6,7,8,9])

data1.append(data2)

Int64Index([1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9], dtype='int64')

data1.difference(data2) #计算两个集合的差集

Int64Index([1, 2, 3], dtype='int64')

data1.intersection(data2)

Int64Index([4, 5], dtype='int64')

data1.union(data2)

Int64Index([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype='int64')

data1.isin(data2)

array([False, False, False,  True,  True])

data3 = data1.delete(0)

data1

Int64Index([1, 2, 3, 4, 5], dtype='int64')

data3

Int64Index([2, 3, 4, 5], dtype='int64')

data4 = data2.drop(7)

data2

Int64Index([4, 5, 6, 7, 8, 9], dtype='int64')

data4

Int64Index([4, 5, 6, 8, 9], dtype='int64')

data2.insert(9,0) #超出长度会自动insert在最后一个位置

Int64Index([4, 5, 6, 7, 8, 9, 0], dtype='int64')

data2.insert(6,1) 

Int64Index([4, 5, 6, 7, 8, 9, 1], dtype='int64')

data1.is_monotonic

True

data1.insert(6,1).is_monotonic

False

data1.is_unique

True

data1.insert(6,1).is_unique

False

data1.unique()

Int64Index([1, 2, 3, 4, 5], dtype='int64')

data1.insert(6,1).unique()

Int64Index([1, 2, 3, 4, 5], dtype='int64')