2.1 pandas数据结构介绍
import pandas as pd
from pandas import Series, DataFrame
import numpy as np
2.1.1 Series(数据+数据标签(即索引))
Series是⼀种类似于⼀维数组的对象(有点像dictionary),它由⼀组数据 (各种NumPy数据类型)以及⼀组与之相关的**数据标签(即索引)**组成。
obj = pd. Series( [ 4 , 7 , - 5 , 3 ] )
obj
0 4
1 7
2 -5
3 3
dtype: int64
索引在左边,默认是从0到N-1,值在右边
2.1.1.1 通过value可以获得值
obj. values
array([ 4, 7, -5, 3], dtype=int64)
2.1.1.2 通过index可以获得索引
obj. index
RangeIndex(start=0, stop=4, step=1)
2.1.1.3 自定义索引
obj2 = pd. Series( [ 4 , 7 , - 5 , 3 ] , index= [ 'd' , 'c' , 'b' , 'a' ] )
obj2
d 4
c 7
b -5
a 3
dtype: int64
obj2. index
Index(['d', 'c', 'b', 'a'], dtype='object')
2.1.1.4 修改索引
obj
0 4
1 7
2 -5
3 3
dtype: int64
obj. index = [ 'Alice' , 'Bob' , 'Eve' , 'Steve' ]
obj
Alice 4
Bob 7
Eve -5
Steve 3
dtype: int64
2.1.1.5 通过索引的方式找值
obj2[ 'a' ]
3
obj2[ [ 'a' , 'b' ] ]
a 3
b -5
dtype: int64
2.1.1.6 进行NumPy运算时候
obj2[ obj2 > 3 ]
d 4
c 7
dtype: int64
obj2* 2
d 8
c 14
b -10
a 6
dtype: int64
np. exp( obj2)
d 54.598150
c 1096.633158
b 0.006738
a 20.085537
dtype: float64
2.1.1.7 判断index是否在Series内
'b' in obj2
True
'e' in obj2
False
2.1.1.8 通过字典创建Series
dict_data = { 'Ohio' : 35000 , 'Texas' : 71000 , 'Oregon' : 16000 , 'Utah' : 5000 }
obj3 = pd. Series( dict_data)
obj3
Ohio 35000
Texas 71000
Oregon 16000
Utah 5000
dtype: int64
states = [ 'California' , 'Ohio' , 'Oregon' , 'Texas' ]
obj4 = pd. Series( dict_data, states)
obj4
California NaN
Ohio 35000.0
Oregon 16000.0
Texas 71000.0
dtype: float64
2.1.1.9 检测缺失数据
pd. isnull( obj4)
California True
Ohio False
Oregon False
Texas False
dtype: bool
pd. notnull( obj4)
California False
Ohio True
Oregon True
Texas True
dtype: bool
2.1.1.10 数据对齐
根据运算的索引标签自动对齐数据 类似数据库的join操作
obj4. isnull( )
California True
Ohio False
Oregon False
Texas False
dtype: bool
obj4. notnull( )
California False
Ohio True
Oregon True
Texas True
dtype: bool
obj3+obj4
obj3+ obj4
California NaN
Ohio 70000.0
Oregon 32000.0
Texas 142000.0
Utah NaN
dtype: float64
2.1.1.10 name属性
obj4. name = 'population'
obj4. index. name = 'state'
obj4
state
California NaN
Ohio 35000.0
Oregon 16000.0
Texas 71000.0
Name: population, dtype: float64
2.1.2 DataFrame
表格型数据结构
一组有序的列,每列可以是不同的值类型 有行索引和列索引 DataFrame内部的数据是以一个或多个二维块存放的
2.1.2.1 创建
通过字典创建 通过list创建,然后再确定index和columns的序号 通过嵌套字典 通过pd.Series组成的字典
data = { 'state' : [ 'Ohio' , 'Ohio' , 'Ohio' , 'Nevada' , 'Nevada' , 'Nevada' ] ,
'year' : [ 2000 , 2001 , 2002 , 2003 , 2004 , 2005 ] ,
'pop' : [ 1.5 , 1.7 , 3.6 , 2.4 , 2.9 , 3.2 ] }
frame = pd. DataFrame( data)
frame
state year pop 0 Ohio 2000 1.5 1 Ohio 2001 1.7 2 Ohio 2002 3.6 3 Nevada 2003 2.4 4 Nevada 2004 2.9 5 Nevada 2005 3.2
data2 = [ [ 'Ohio' , 'Ohio' , 'Ohio' , 'Nevada' , 'Nevada' , 'Nevada' ] ,
[ 2000 , 2001 , 2002 , 2003 , 2004 , 2005 ] ,
[ 1.5 , 1.7 , 3.6 , 2.4 , 2.9 , 3.2 ] ]
pd. DataFrame( data2, index= [ 'state' , 'year' , 'pop' ] , columns= [ 1 , 2 , 3 , 4 , 5 , 6 ] )
1 2 3 4 5 6 state Ohio Ohio Ohio Nevada Nevada Nevada year 2000 2001 2002 2003 2004 2005 pop 1.5 1.7 3.6 2.4 2.9 3.2
data3 = { 'Nevada' : { 2001 : 2.4 , 2002 : 2.9 } ,
'Ohio' : { 2000 : 1.5 , 2001 : 1.7 , 2002 : 3.6 }
}
frame3 = pd. DataFrame( data3)
frame3
Nevada Ohio 2001 2.4 1.7 2002 2.9 3.6 2000 NaN 1.5
pdata = { 'Ohio' : frame3[ 'Ohio' ] [ : - 1 ] ,
'Nevada' : frame3[ 'Nevada' ] [ : 2 ] }
pd. DataFrame( pdata)
Ohio Nevada 2001 1.7 2.4 2002 3.6 2.9
2.1.2.2 head()/tail()
对于比较庞大的数据,head()会选取0~4行(共5行数据) tail()会选取倒数5行
frame. head( )
state year pop 0 Ohio 2000 1.5 1 Ohio 2001 1.7 2 Ohio 2002 3.6 3 Nevada 2003 2.4 4 Nevada 2004 2.9
frame. tail( )
state year pop 1 Ohio 2001 1.7 2 Ohio 2002 3.6 3 Nevada 2003 2.4 4 Nevada 2004 2.9 5 Nevada 2005 3.2
2.1.2.3 指定列序列
pd. DataFrame( data, columns= [ 'year' , 'state' , 'pop' ] )
year state pop 0 2000 Ohio 1.5 1 2001 Ohio 1.7 2 2002 Ohio 3.6 3 2003 Nevada 2.4 4 2004 Nevada 2.9 5 2005 Nevada 3.2
2.1.2.4 扩增列无数据会用NaN填充
frame2 = pd. DataFrame( data, columns= [ 'year' , 'state' , 'pop' , 'debt' ] , index= [ 'one' , 'two' , 'three' , 'four' , 'five' , 'six' ] )
frame2
year state pop debt one 2000 Ohio 1.5 NaN two 2001 Ohio 1.7 NaN three 2002 Ohio 3.6 NaN four 2003 Nevada 2.4 NaN five 2004 Nevada 2.9 NaN six 2005 Nevada 3.2 NaN
2.1.2.5 获取某行/列
frame2. state
one Ohio
two Ohio
three Ohio
four Nevada
five Nevada
six Nevada
Name: state, dtype: object
frame2[ 'state' ]
one Ohio
two Ohio
three Ohio
four Nevada
five Nevada
six Nevada
Name: state, dtype: object
frame2. loc[ 'one' ]
year 2000
state Ohio
pop 1.5
debt -1.2
Name: one, dtype: object
2.1.2.6 赋值
frame2[ 'debt' ] = np. arange( 1 , 7 )
frame2
year state pop debt one 2000 Ohio 1.5 1 two 2001 Ohio 1.7 2 three 2002 Ohio 3.6 3 four 2003 Nevada 2.4 4 five 2004 Nevada 2.9 5 six 2005 Nevada 3.2 6
frame2[ 'debt' ] = [ 4 , 4 , 4 , 4 , 4 , 4 ]
frame2
year state pop debt one 2000 Ohio 1.5 4 two 2001 Ohio 1.7 4 three 2002 Ohio 3.6 4 four 2003 Nevada 2.4 4 five 2004 Nevada 2.9 4 six 2005 Nevada 3.2 4
frame2[ 'debt' ] = 3
frame2
year state pop debt one 2000 Ohio 1.5 3 two 2001 Ohio 1.7 3 three 2002 Ohio 3.6 3 four 2003 Nevada 2.4 3 five 2004 Nevada 2.9 3 six 2005 Nevada 3.2 3
val = pd. Series( [ - 1.2 , - 1.5 , - 1.7 ] , index = [ 'one' , 'three' , 'six' ] )
val
one -1.2
three -1.5
six -1.7
dtype: float64
frame2[ 'debt' ] = val
frame2
year state pop debt one 2000 Ohio 1.5 -1.2 two 2001 Ohio 1.7 NaN three 2002 Ohio 3.6 -1.5 four 2003 Nevada 2.4 NaN five 2004 Nevada 2.9 NaN six 2005 Nevada 3.2 -1.7
2.1.2.7 新建某行/列
列:frame[column];pd.DataFrame(frame, columns = []) 行:frame.loc[index];pd.DataFrame(frame, index = [])
frame2[ 'estern' ] = frame2. state == 'Ohio'
frame2
year state pop debt estern one 2000 Ohio 1.5 -1.2 True two 2001 Ohio 1.7 NaN True three 2002 Ohio 3.6 -1.5 True four 2003 Nevada 2.4 NaN False five 2004 Nevada 2.9 NaN False six 2005 Nevada 3.2 -1.7 False
frame2. loc[ 'seven' ] = [ 2005 , 'Nevada' , '3.0' , '-1.7' , False ]
frame2
year state pop debt estern one 2000 Ohio 1.5 -1.2 True two 2001 Ohio 1.7 NaN True three 2002 Ohio 3.6 -1.5 True four 2003 Nevada 2.4 NaN False five 2004 Nevada 2.9 NaN False six 2005 Nevada 3.2 -1.7 False seven 2005 Nevada 3.0 -1.7 False
frame3
Nevada Ohio 2001 2.4 1.7 2002 2.9 3.6 2000 NaN 1.5
pd. DataFrame( frame3, index= [ 2001 , 2002 , 2003 ] )
Nevada Ohio 2001 2.4 1.7 2002 2.9 3.6 2003 NaN NaN
2.1.2.8 删除行/列
drop()可以删除行/列,默认axis=0(行),不是实际删除 del函数可以删除列,是实际删除
frame2. drop( 'six' )
year state pop debt estern one 2000 Ohio 1.5 -1.2 True two 2001 Ohio 1.7 NaN True three 2002 Ohio 3.6 -1.5 True four 2003 Nevada 2.4 NaN False five 2004 Nevada 2.9 NaN False seven 2005 Nevada 3.0 -1.7 False
frame2. drop( 'estern' , axis= 1 )
year state pop debt one 2000 Ohio 1.5 -1.2 two 2001 Ohio 1.7 NaN three 2002 Ohio 3.6 -1.5 four 2003 Nevada 2.4 NaN five 2004 Nevada 2.9 NaN six 2005 Nevada 3.2 -1.7 seven 2005 Nevada 3.0 -1.7
del frame2[ 'estern' ]
frame2
year state pop debt one 2000 Ohio 1.5 -1.2 two 2001 Ohio 1.7 NaN three 2002 Ohio 3.6 -1.5 four 2003 Nevada 2.4 NaN five 2004 Nevada 2.9 NaN six 2005 Nevada 3.2 -1.7 seven 2005 Nevada 3.0 -1.7
2.1.2.9 转置
frame3
Nevada Ohio 2001 2.4 1.7 2002 2.9 3.6 2000 NaN 1.5
frame3. T
2001 2002 2000 Nevada 2.4 2.9 NaN Ohio 1.7 3.6 1.5
pd. DataFrame( frame3, index = [ 2001 , 2002 , 2003 ] )
Nevada Ohio 2001 2.4 1.7 2002 2.9 3.6 2003 NaN NaN
2.1.2.10 获取行,列,值
frame3. index
Int64Index([2001, 2002, 2000], dtype='int64')
frame3. columns
Index(['Nevada', 'Ohio'], dtype='object')
frame3. values
array([[2.4, 1.7],
[2.9, 3.6],
[nan, 1.5]])
frame2. values
array([[2000, 'Ohio', 1.5, -1.2],
[2001, 'Ohio', 1.7, nan],
[2002, 'Ohio', 3.6, -1.5],
[2003, 'Nevada', 2.4, nan],
[2004, 'Nevada', 2.9, nan],
[2005, 'Nevada', 3.2, -1.7],
[2005, 'Nevada', '3.0', '-1.7']], dtype=object)
2.1.3 索引对象 (Index)
类似数组 类似集合,但可以包含重复元素
obj = pd. Series( range ( 3 ) , index= [ 'a' , 'b' , 'c' ] )
obj. index
Index(['a', 'b', 'c'], dtype='object')
obj. index[ 1 : ]
Index(['b', 'c'], dtype='object')
2.1.3.1 index不可变
obj. index[ 0 ] = 'd'
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-142-d2bd8d3dd4c5> in <module>
----> 1 obj.index[0]='d'
F:\Anaconda3\lib\site-packages\pandas\core\indexes\base.py in __setitem__(self, key, value)
3908
3909 def __setitem__(self, key, value):
-> 3910 raise TypeError("Index does not support mutable operations")
3911
3912 def __getitem__(self, key):
TypeError: Index does not support mutable operations
2.1.3.2 index可共享
labels = pd. Index( np. arange( 3 ) )
labels
Int64Index([0, 1, 2], dtype='int64')
obj2 = pd. Series( [ 'a' , 'b' , 'c' ] , index= labels)
obj2
0 a
1 b
2 c
dtype: object
obj2. index is labels
True
2.1.3.3 Index可包含重复标签
duplicates = pd. Index( [ 'abc' , 'abc' , 'lalala' , 'Jeff' ] )
duplicates
Index(['abc', 'abc', 'lalala', 'Jeff'], dtype='object')
2.1.3.4 Index的方法和属性
方法 说明 append 把当前Index对象和另一个Index连接起来 difference 计算两个集合的差集 intersection 计算两个集合的交集 union 计算两个集合的并集,注意 :union和append得到的结果不一定一样 isin 返回的是bool类型的对应索引处元素是否在另一个Index内 delete 删除索引处的元素,并返回新的Index,旧Index索引处元素并未被删除 drop 删除值,并返回新的Index(只有当值是unique的时候可以使用) insert 将元素插入索引处,会覆盖原有值,并得到新的Index is_monotonic 当每个元素全都大于前一个元素的时候,返回True is_unique Index内没有重复元素就返回True unique 返回没有重复的原Index中的所有元素组成的新Index,比如[1,2,3,4,1,2,3]->[1,2,3,4]
下面是按顺序的测试:
data1 = pd. Index( [ 1 , 2 , 3 , 4 , 5 ] )
data2 = pd. Index( [ 4 , 5 , 6 , 7 , 8 , 9 ] )
data1. append( data2)
Int64Index([1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9], dtype='int64')
data1. difference( data2)
Int64Index([1, 2, 3], dtype='int64')
data1. intersection( data2)
Int64Index([4, 5], dtype='int64')
data1. union( data2)
Int64Index([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype='int64')
data1. isin( data2)
array([False, False, False, True, True])
data3 = data1. delete( 0 )
data1
Int64Index([1, 2, 3, 4, 5], dtype='int64')
data3
Int64Index([2, 3, 4, 5], dtype='int64')
data4 = data2. drop( 7 )
data2
Int64Index([4, 5, 6, 7, 8, 9], dtype='int64')
data4
Int64Index([4, 5, 6, 8, 9], dtype='int64')
data2. insert( 9 , 0 )
Int64Index([4, 5, 6, 7, 8, 9, 0], dtype='int64')
data2. insert( 6 , 1 )
Int64Index([4, 5, 6, 7, 8, 9, 1], dtype='int64')
data1. is_monotonic
True
data1. insert( 6 , 1 ) . is_monotonic
False
data1. is_unique
True
data1. insert( 6 , 1 ) . is_unique
False
data1. unique( )
Int64Index([1, 2, 3, 4, 5], dtype='int64')
data1. insert( 6 , 1 ) . unique( )
Int64Index([1, 2, 3, 4, 5], dtype='int64')