Python零基础投喂（Pandas基础）

PANDAS 基础

import numpy as np
import pandas as pd

pd.__version__

'1.1.5'

2.1 文件的读取和写入

2.1.1 文件读取

# pandas 可以读取的文件格式有很多，这里主要介绍读取 csv, excel, txt 文件
df_csv = pd.read_csv('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_csv.csv')

df_csv

	col1	col2	col3	col4	col5
0	2	a	1.4	apple	2020/1/1
1	3	b	3.4	banana	2020/1/2
2	6	c	2.5	orange	2020/1/5
3	5	d	3.2	lemon	2020/1/7

df_txt = pd.read_table('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_table.txt')
df_txt

	col1	col2	col3	col4
0	2	a	1.4	apple 2020/1/1
1	3	b	3.4	banana 2020/1/2
2	6	c	2.5	orange 2020/1/5
3	5	d	3.2	lemon 2020/1/7

df_excel = pd.read_excel('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_excel.xlsx')
df_excel

	col1	col2	col3	col4	col5
0	2	a	1.4	apple	2020/1/1
1	3	b	3.4	banana	2020/1/2
2	6	c	2.5	orange	2020/1/5
3	5	d	3.2	lemon	2020/1/7

# header=None 表示第一行不作为列名
pd.read_table('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_table.txt',header=None)

	0	1	2	3
0	col1	col2	col3	col4
1	2	a	1.4	apple 2020/1/1
2	3	b	3.4	banana 2020/1/2
3	6	c	2.5	orange 2020/1/5
4	5	d	3.2	lemon 2020/1/7

# index_col 表示把某一列或几列作为索引
pd.read_csv('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_csv.csv', index_col=['col1', 'col2'])

		col3	col4	col5
col1	col2
2	a	1.4	apple	2020/1/1
3	b	3.4	banana	2020/1/2
6	c	2.5	orange	2020/1/5
5	d	3.2	lemon	2020/1/7

# usecols 表示读取列的集合，默认读取所有的列
pd.read_table('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_table.txt', usecols=['col1', 'col2'])

	col1	col2
0	2	a
1	3	b
2	6	c
3	5	d

# parse_dates 表示需要转化为时间的列，
pd.read_csv('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_csv.csv', parse_dates=['col5'])

	col1	col2	col3	col4	col5
0	2	a	1.4	apple	2020-01-01
1	3	b	3.4	banana	2020-01-02
2	6	c	2.5	orange	2020-01-05
3	5	d	3.2	lemon	2020-01-07

 # ，nrows 表示读取的数据行数
pd.read_excel('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_excel.xlsx', nrows=2)

	col1	col2	col3	col4	col5
0	2	a	1.4	apple	2020/1/1
1	3	b	3.4	banana	2020/1/2

'''在读取 txt 文件时，经常遇到分隔符非空格的情况，read_table 有一个分割参数 sep ，它使得用户可以自定
义分割符号，进行 txt 数据的读取。例如，下面的读取的表以 |||| 为分割：'''
pd.read_table('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_table_special_sep.txt')

	col1 |||| col2
0	TS |||| This is an apple.
1	GQ |||| My name is Bob.
2	WT |||| Well done!
3	PT |||| May I help you?

# 上面的结果显然不是理想的，这时可以使用 sep ，同时需要指定引擎为 python ：sep 是正则参数,在使用 read_table 的时候需要注意，参数 sep 中使用的是正则表达式，因此需要对 | 进行转义
变成 \| ，否则无法读取到正确的结果。
pd.read_table('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_table_special_sep.txt',sep=' \|\|\|\| ', engine='python')

	col1	col2
0	TS	This is an apple.
1	GQ	My name is Bob.
2	WT	Well done!
3	PT	May I help you?

2.1.2 数据写入

#一般在数据写入中，最常用的操作是把 index 设置为 False ，特别当索引没有特殊意义的时候，这样的行为能把索引在保存的时候去除。
df_csv.to_csv('E:/DW学习/Python 学习代码/joyful-pandas-master/data/my_csv_saved.csv', index=False)
df_excel.to_excel('data/my_excel_saved.xlsx', index=False)

# pandas 中没有定义 to_table 函数，但是 to_csv 可以保存为 txt 文件，并且允许自定义分隔符，常用制表符\t 分割：
df_txt.to_csv('data/my_txt_saved.txt', sep='\t', index=False)

# 如果想要把表格快速转换为 markdown 和 latex 语言，可以使用 to_markdown 和 to_latex 函数，此处需要安装 tabulate 包。
! pip install tabulate #jupyter 安装包的语句

Collecting tabulate
  Downloading tabulate-0.8.7-py3-none-any.whl (24 kB)
Installing collected packages: tabulate
Successfully installed tabulate-0.8.7

print(df_csv.to_markdown())

|    |   col1 | col2   |   col3 | col4   | col5     |
|---:|-------:|:-------|-------:|:-------|:---------|
|  0 |      2 | a      |    1.4 | apple  | 2020/1/1 |
|  1 |      3 | b      |    3.4 | banana | 2020/1/2 |
|  2 |      6 | c      |    2.5 | orange | 2020/1/5 |
|  3 |      5 | d      |    3.2 | lemon  | 2020/1/7 |

 print(df_csv.to_latex())

\begin{tabular}{lrlrll}
\toprule
{} &  col1 & col2 &  col3 &    col4 &      col5 \\
\midrule
0 &     2 &    a &   1.4 &   apple &  2020/1/1 \\
1 &     3 &    b &   3.4 &  banana &  2020/1/2 \\
2 &     6 &    c &   2.5 &  orange &  2020/1/5 \\
3 &     5 &    d &   3.2 &   lemon &  2020/1/7 \\
\bottomrule
\end{tabular}

2.2 基本数据结构

# pandas 中具有两种基本的数据存储结构，存储一维 values 的 Series 和存储二维 values 的 DataFrame ，在这两种结构上定义了很多的属性和方法。

2.2.1 Series

'''Series 一般由四个部分组成，分别是序列的值 data 、索引 index 、存储类型 dtype 、序列的名字 name 。其
中，索引也可以指定它的名字，默认为空。'''
s=pd.Series(data=[100,'a',{'dict':5}],
            index=pd.Index(['idi',20,'third'],name='my_idx'),
            dtype='object',
            name='my_name')
s

my_idx
idi              100
20                 a
third    {'dict': 5}
Name: my_name, dtype: object

object 类型

'''object 代表了一种混合类型，正如上面的例子中存储了整数、字符串以及 Python 的字典数据结
构。此外，目前 pandas 把纯字符串序列也默认认为是一种 object 类型的序列，但它也可以用
string 类型存储
对于这些属性，可以通过 . 的方式来获取：
'''

s.values

array([100, 'a', {'dict': 5}], dtype=object)

s.index

Index(['idi', 20, 'third'], dtype='object', name='my_idx')

s.dtype

dtype('O')

s.name

'my_name'

s.shape

(3,)

#索引是 pandas 中最重要的概念之一，如果想要取出单个索引对应的值，可以通过 [index_item] 可以取出。
s['third']

{'dict': 5}

2.2.2 DataFrame

# DataFrame 在 Series 的基础上增加了列索引，一个数据框可以由二维的 data 与行列索引来构造：
data = [[1, 'a', 1.2], [2, 'b', 2.2], [3, 'c', 3.2]]
df=pd.DataFrame(data=data,
                index =['row_%d'%i for i in range(3)],
                columns=['col_0', 'col_1', 'col_2'])

df

	col_0	col_1	col_2
row_0	1	a	1.2
row_1	2	b	2.2
row_2	3	c	3.2

df=pd.DataFrame(data=data,
                index =['row_%d'%i for i in range(3)],
                columns=['col_%d'%i for i in range(3)])
df

	col_0	col_1	col_2
row_0	1	a	1.2
row_1	2	b	2.2
row_2	3	c	3.2

# 但一般而言，更多的时候会采用从列索引名到数据的映射来构造数据框，同时再加上行索引：
df=pd.DataFrame(data={'col_0':[1,2,3],'col_1':list('abc'),
                       'col_2':[1.2,2.2,3.2]},
                   index=['row_%d'%i for i in range(3)])
df

	col_0	col_1	col_2
row_0	1	a	1.2
row_1	2	b	2.2
row_2	3	c	3.2

# 由于这种映射关系，在 DataFrame 中可以用 [col_name] 与 [col_list] 来取出相应的列与由多个列组成的表，结果分别为 Series 和 DataFrame ：
df['col_0']

row_0    1
row_1    2
row_2    3
Name: col_0, dtype: int64

df[['col_0','col_1']]

	col_0	col_1
row_0	1	a
row_1	2	b
row_2	3	c

# 与 Series 类似，在数据框中同样可以取出相应的属性：
df.values

array([[1, 'a', 1.2],
       [2, 'b', 2.2],
       [3, 'c', 3.2]], dtype=object)

df.index

Index(['row_0', 'row_1', 'row_2'], dtype='object')

df.columns

Index(['col_0', 'col_1', 'col_2'], dtype='object')

df.dtypes

col_0      int64
col_1     object
col_2    float64
dtype: object

df.shape

(3, 3)

# 通过 .T 可以把 DataFrame 进行转置：
df.T

	row_0	row_1	row_2
col_0	1	2	3
col_1	a	b	c
col_2	1.2	2.2	3.2

2.3 常用基本函数

# 为了进行举例说明，在接下来的部分和其余章节都将会使用一份 learn_pandas.csv 的虚拟数据集，它记录了
# 四所学校学生的体测个人信息。
df=pd.read_csv('E:/DW学习/Python 学习代码/joyful-pandas-master/data/learn_pandas.csv')

df.columns
#列名依次代表学校、年级、姓名、性别、身高、体重、是否为转系生、体测场次、测试时间、1000 米成绩，本章只需使用其中的前七列。

Index(['School', 'Grade', 'Name', 'Gender', 'Height', 'Weight', 'Transfer',
       'Test_Number', 'Test_Date', 'Time_Record'],
      dtype='object')

df

	School	Grade	Name	Gender	Height	Weight	Transfer	Test_Number	Test_Date	Time_Record
0	Shanghai Jiao Tong University	Freshman	Gaopeng Yang	Female	158.9	46.0	N	1	2019/10/5	0:04:34
1	Peking University	Freshman	Changqiang You	Male	166.5	70.0	N	1	2019/9/4	0:04:20
2	Shanghai Jiao Tong University	Senior	Mei Sun	Male	188.9	89.0	N	2	2019/9/12	0:05:22
3	Fudan University	Sophomore	Xiaojuan Sun	Female	NaN	41.0	N	2	2020/1/3	0:04:08
4	Fudan University	Sophomore	Gaojuan You	Male	174.0	74.0	N	2	2019/11/6	0:05:22
...	...	...	...	...	...	...	...	...	...	...
195	Fudan University	Junior	Xiaojuan Sun	Female	153.9	46.0	N	2	2019/10/17	0:04:31
196	Tsinghua University	Senior	Li Zhao	Female	160.9	50.0	N	3	2019/9/22	0:04:03
197	Shanghai Jiao Tong University	Senior	Chengqiang Chu	Female	153.9	45.0	N	1	2020/1/5	0:04:48
198	Shanghai Jiao Tong University	Senior	Chengmei Shen	Male	175.3	71.0	N	2	2020/1/7	0:04:58
199	Tsinghua University	Sophomore	Chunpeng Lv	Male	155.7	51.0	N	1	2019/11/6	0:05:05

200 rows × 10 columns

df = df[df.columns[:7]]
df

	School	Grade	Name	Gender	Height	Weight	Transfer
0	Shanghai Jiao Tong University	Freshman	Gaopeng Yang	Female	158.9	46.0	N
1	Peking University	Freshman	Changqiang You	Male	166.5	70.0	N
2	Shanghai Jiao Tong University	Senior	Mei Sun	Male	188.9	89.0	N
3	Fudan University	Sophomore	Xiaojuan Sun	Female	NaN	41.0	N
4	Fudan University	Sophomore	Gaojuan You	Male	174.0	74.0	N
...	...	...	...	...	...	...	...
195	Fudan University	Junior	Xiaojuan Sun	Female	153.9	46.0	N
196	Tsinghua University	Senior	Li Zhao	Female	160.9	50.0	N
197	Shanghai Jiao Tong University	Senior	Chengqiang Chu	Female	153.9	45.0	N
198	Shanghai Jiao Tong University	Senior	Chengmei Shen	Male	175.3	71.0	N
199	Tsinghua University	Sophomore	Chunpeng Lv	Male	155.7	51.0	N

200 rows × 7 columns

2.3.1 汇总函数

# head, tail 函数分别表示返回表或者序列的前 n 行和后 n 行，其中 n 默认为 5：
df.head(2)

	School	Grade	Name	Gender	Height	Weight	Transfer
0	Shanghai Jiao Tong University	Freshman	Gaopeng Yang	Female	158.9	46.0	N
1	Peking University	Freshman	Changqiang You	Male	166.5	70.0	N

df.tail(3)

	School	Grade	Name	Gender	Height	Weight	Transfer
197	Shanghai Jiao Tong University	Senior	Chengqiang Chu	Female	153.9	45.0	N
198	Shanghai Jiao Tong University	Senior	Chengmei Shen	Male	175.3	71.0	N
199	Tsinghua University	Sophomore	Chunpeng Lv	Male	155.7	51.0	N

# info, describe 分别返回表的信息概况和表中数值列对应的主要统计量：
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 200 entries, 0 to 199
Data columns (total 7 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   School    200 non-null    object 
 1   Grade     200 non-null    object 
 2   Name      200 non-null    object 
 3   Gender    200 non-null    object 
 4   Height    183 non-null    float64
 5   Weight    189 non-null    float64
 6   Transfer  188 non-null    object 
dtypes: float64(2), object(5)
memory usage: 11.1+ KB

df.describe()

	Height	Weight
count	183.000000	189.000000
mean	163.218033	55.015873
std	8.608879	12.824294
min	145.400000	34.000000
25%	157.150000	46.000000
50%	161.900000	51.000000
75%	167.500000	65.000000
max	193.900000	89.000000

# 备注：info, describe 只能实现较少信息的展示，如果想要对一份数据集进行全面且有效的观察，特别是
# 在列较多的情况下，推荐使用 pandas-profiling 包

2.3.2 特征统计函数

# 在 Series 和 DataFrame 上定义了许多统计函数，最常见的是 sum, mean, median, var, std, max, min 。例
# 如，选出身高和体重列进行演示：
df_demo = df[['Height', 'Weight']]
df_demo.mean()

Height    163.218033
Weight     55.015873
dtype: float64

 df_demo.max()

Height    193.9
Weight     89.0
dtype: float64

#此外，需要介绍的是 quantile, count, idxmax 这三个函数，它们分别返回的是分位数、非缺失值个数、最大
#值对应的索引：
df_demo.quantile(0.75)

Height    167.5
Weight     65.0
Name: 0.75, dtype: float64

df_demo.count()

Height    183
Weight    189
dtype: int64

df_demo.idxmax()

Height    193
Weight      2
dtype: int64

# 上面这些所有的函数，由于操作后返回的是标量，所以又称为聚合函数，它们有一个公共参数 axis ，默认为
# 0 代表逐列聚合，如果设置为 1 则表示逐行聚合：
df_demo.mean(axis=1).head(10)

0    102.45
1    118.25
2    138.95
3     41.00
4    124.00
5    104.50
6    107.25
7    105.95
8    105.50
9    164.80
dtype: float64

df_demo

	Height	Weight
0	158.9	46.0
1	166.5	70.0
2	188.9	89.0
3	NaN	41.0
4	174.0	74.0
...	...	...
195	153.9	46.0
196	160.9	50.0
197	153.9	45.0
198	175.3	71.0
199	155.7	51.0

200 rows × 2 columns

2.3.3 唯一值函数

# 对序列使用 unique 和 nunique 可以分别得到其唯一值组成的列表和唯一值的个数：
df['School'].unique()

array(['Shanghai Jiao Tong University', 'Peking University',
       'Fudan University', 'Tsinghua University'], dtype=object)

df['School'].nunique()

4

# value_counts 可以得到唯一值和其对应出现的频数：
df['School'].value_counts()

Tsinghua University              69
Shanghai Jiao Tong University    57
Fudan University                 40
Peking University                34
Name: School, dtype: int64

'''如果想要观察多个列组合的唯一值，可以使用 drop_duplicates 。其中的关键参数是 keep ，默认值 first 表
示每个组合保留第一次出现的所在行，last 表示保留最后一次出现的所在行，False 表示把所有重复组合所
在的行剔除。'''
# duplicates 副本; 复制品; 完全一样的东西;
df_demo = df[['Gender','Transfer','Name']]
df_demo

	Gender	Transfer	Name
0	Female	N	Gaopeng Yang
1	Male	N	Changqiang You
2	Male	N	Mei Sun
3	Female	N	Xiaojuan Sun
4	Male	N	Gaojuan You
...	...	...	...
195	Female	N	Xiaojuan Sun
196	Female	N	Li Zhao
197	Female	N	Chengqiang Chu
198	Male	N	Chengmei Shen
199	Male	N	Chunpeng Lv

200 rows × 3 columns

df_demo.drop_duplicates(['Gender', 'Transfer'])

	Gender	Transfer	Name
0	Female	N	Gaopeng Yang
1	Male	N	Changqiang You
12	Female	NaN	Peng You
21	Male	NaN	Xiaopeng Shen
36	Male	Y	Xiaojuan Qin
43	Female	Y	Gaoli Feng

 df_demo.drop_duplicates(['Gender', 'Transfer'], keep='last')

	Gender	Transfer	Name
147	Male	NaN	Juan You
150	Male	Y	Chengpeng You
169	Female	Y	Chengquan Qin
194	Female	NaN	Yanmei Qian
197	Female	N	Chengqiang Chu
199	Male	N	Chunpeng Lv

df_demo.drop_duplicates(['Name', 'Gender'],keep=False).head(10) 

	Gender	Transfer	Name
0	Female	N	Gaopeng Yang
1	Male	N	Changqiang You
2	Male	N	Mei Sun
4	Male	N	Gaojuan You
5	Female	N	Xiaoli Qian
7	Female	N	Gaoqiang Qian
8	Female	N	Changli Zhang
9	Female	N	Juan Xu
10	Male	N	Xiaopeng Zhou
11	Female	N	Xiaoquan Lv

df['School'].duplicated().head() # 在 Series 上也可以使用

0    False
1    False
2     True
3    False
4     True
Name: School, dtype: bool

2.3.4 替换函数

'''一般而言，替换操作是针对某一个列进行的，因此下面的例子都以 Series 举例。pandas 中的替换函数可以
归纳为三类：映射替换、逻辑替换、数值替换。其中映射替换包含 replace 方法、第八章中的 str.replace 方
法以及第九章中的 cat.codes 方法，此处介绍 replace 的用法。
'''
# 在 replace 中，可以通过字典构造，或者传入两个列表来进行替换
df['Gender']

0      Female
1        Male
2        Male
3      Female
4        Male
        ...  
195    Female
196    Female
197    Female
198      Male
199      Male
Name: Gender, Length: 200, dtype: object

df['Gender'].replace({'Female':0,'Male':1}).head()

0    0
1    1
2    1
3    0
4    1
Name: Gender, dtype: int64

df['Gender'].replace(['Female','Male'],[0,1]).head()

0    0
1    1
2    1
3    0
4    1
Name: Gender, dtype: int64

'''另外，replace 还有一种特殊的方向替换，指定 method 参数为 ffill 则为用前面一个最近的未被替换的值进行
替换，bfill 则使用后面最近的未被替换的值进行替换。从下面的例子可以看到，它们的结果是不同的：'''
s = pd.Series(['a', 1, 'b', 2, 1, 1, 'a'])
s

0    a
1    1
2    b
3    2
4    1
5    1
6    a
dtype: object

s.replace([1,2],method='ffill')

0    a
1    a
2    b
3    b
4    b
5    b
6    a
dtype: object

s.replace([1, 2], method='bfill')

0    a
1    b
2    b
3    a
4    a
5    a
6    a
dtype: object

# 逻辑替换包括了 where 和 mask ，这两个函数是完全对称的：where 函数在传入条件为 False 的对应行进行
# 替换，而 mask 在传入条件为 True 的对应行进行替换，当不指定替换值时，替换为缺失值。
s = pd.Series([-1, 1.2345, 100, -50])
s

0     -1.0000
1      1.2345
2    100.0000
3    -50.0000
dtype: float64

s.where(s<0)

0    -1.0
1     NaN
2     NaN
3   -50.0
dtype: float64

s.where(s<0, 100)

0     -1.0
1    100.0
2    100.0
3    -50.0
dtype: float64

s.mask(s<0)  #mask 掩饰; 掩藏;面具; 面膜

0         NaN
1      1.2345
2    100.0000
3         NaN
dtype: float64

 s.mask(s<0, -50)

0    -50.0000
1      1.2345
2    100.0000
3    -50.0000
dtype: float64

s_condition= pd.Series([True,False,False,True],index=s.index)
s_condition

0     True
1    False
2    False
3     True
dtype: bool

s.mask(s_condition, -50)

0    -50.0000
1      1.2345
2    100.0000
3    -50.0000
dtype: float64

# s = pd.Series([-1, 1.2345, 100, -50])
s

s.round(2)

0     -1.00
1      1.23
2    100.00
3    -50.00
dtype: float64

s.abs()

0      1.0000
1      1.2345
2    100.0000
3     50.0000
dtype: float64

 s.clip(0, 2) # 前两个数分别表示上下截断边界

0    0.0000
1    1.2345
2    2.0000
3    0.0000
dtype: float64

2.3.5 排序函数

'''排序共有两种方式，其一为值排序，其二为索引排序，对应的函数是 sort_values 和 sort_index 。
为了演示排序函数，下面先利用 set_index 方法把年级和姓名两列作为索引'''
df_demo=df[['Grade','Name','Height','Weight']].set_index(['Grade','Name'])
df_demo

		Height	Weight
Grade	Name
Freshman	Gaopeng Yang	158.9	46.0
	Changqiang You	166.5	70.0
Senior	Mei Sun	188.9	89.0
Sophomore	Xiaojuan Sun	NaN	41.0
	Gaojuan You	174.0	74.0
...	...	...	...
Junior	Xiaojuan Sun	153.9	46.0
Senior	Li Zhao	160.9	50.0
	Chengqiang Chu	153.9	45.0
	Chengmei Shen	175.3	71.0
Sophomore	Chunpeng Lv	155.7	51.0

200 rows × 2 columns

# 对身高进行排序，默认参数 ascending=True 为升序：
df_demo.sort_values('Height',ascending=False).head()

		Height	Weight
Grade	Name
Senior	Xiaoqiang Qin	193.9	79.0
	Mei Sun	188.9	89.0
	Gaoli Zhao	186.5	83.0
Freshman	Qiang Han	185.3	87.0
Senior	Qiang Zheng	183.9	87.0

# 在排序中，经常遇到多列排序的问题，比如在体重相同的情况下，对身高进行排序，并且保持身高降序排列，
# 体重升序排列：
df_demo.sort_values(['Weight','Height'],ascending=[True,False]).head()

		Height	Weight
Grade	Name
Sophomore	Peng Han	147.8	34.0
Senior	Gaomei Lv	147.3	34.0
Junior	Xiaoli Chu	145.4	34.0
Sophomore	Qiang Zhou	150.5	36.0
Freshman	Yanqiang Xu	152.4	38.0

# 索引排序的用法和值排序完全一致，只不过元素的值在索引中，此时需要指定索引层的名字或者层号，用参
# 数 level 表示。另外，需要注意的是字符串的排列顺序由字母顺序决定。
df_demo.sort_values(['Grade','Name'],ascending=[True,False]).head(100)

		Height	Weight
Grade	Name
Freshman	Yanquan Wang	163.5	55.0
	Yanqiang Xu	152.4	38.0
	Yanqiang Feng	162.3	51.0
	Yanpeng Lv	NaN	65.0
	Yanli Zhang	165.1	52.0
...	...	...	...
Junior	Chunqiang Chu	168.6	72.0
	Chunmei Han	153.2	44.0
	Chunjuan Zhang	158.9	47.0
	Chunfeng Zhao	173.4	72.0
	Chengquan Shi	160.8	51.0

100 rows × 2 columns

2.3.6 apply 方法

'''apply 方法常用于 DataFrame 的行迭代或者列迭代，它的 axis 含义与第 2 小节中的统计聚合函数一致，apply
的参数往往是一个以序列为输入的函数。例如对于 .mean() ，使用 apply 可以如下地写出：'''
df_demo = df[['Height', 'Weight']]
df_demo

	Height	Weight
0	158.9	46.0
1	166.5	70.0
2	188.9	89.0
3	NaN	41.0
4	174.0	74.0
...	...	...
195	153.9	46.0
196	160.9	50.0
197	153.9	45.0
198	175.3	71.0
199	155.7	51.0

200 rows × 2 columns

def my_mean(x):
    res=x.mean()
    return res

df_demo.apply(my_mean)

Height    163.218033
Weight     55.015873
dtype: float64

# 同样的，可以利用 lambda 表达式使得书写简洁，这里的 x 就指代被调用的 df_demo 表中逐个输入的序列：
df_demo.apply(lambda x:x.mean())

Height    163.218033
Weight     55.015873
dtype: float64

# 若指定 axis=1 ，那么每次传入函数的就是行元素组成的 Series ，其结果与之前的逐行均值结果一致。
df_demo.apply(lambda x:x.mean(),axis=1)

0      102.45
1      118.25
2      138.95
3       41.00
4      124.00
        ...  
195     99.95
196    105.45
197     99.45
198    123.15
199    103.35
Length: 200, dtype: float64

'''这里再举一个例子：mad 函数返回的是一个序列中偏离该序列均值的绝对值大小的均值，例如序列 1,3,7,10
中，均值为 5.25，每一个元素偏离的绝对值为 4.25,2.25,1.75,4.75，这个偏离序列的均值为 3.25。现在利用
apply 计算升高和体重的 mad 指标：
'''
df_demo.apply(lambda x:(x-x.mean()).abs().mean())

Height     6.707229
Weight    10.391870
dtype: float64

# 这与使用内置的 mad 函数计算结果一致：
df_demo.mad()

Height     6.707229
Weight    10.391870
dtype: float64

'''谨慎使用 apply
得益于传入自定义函数的处理，apply 的自由度很高，但这是以性能为代价的。一般而言，使用
pandas 的内置函数处理和 apply 来处理同一个任务，其速度会相差较多，因此只有在确实存在
自定义需求的情境下才考虑使用 apply'''

2.4 窗口对象

# pandas 中有 3 类窗口，分别是滑动窗口 rolling 、扩张窗口 expanding 以及指数加权窗口 ewm 。

2.4.1 滑窗对象

# 要使用滑窗函数，就必须先要对一个序列使用 .rolling 得到滑窗对象，其最重要的参数为窗口大小 window 。
s=pd.Series([1,2,3,4,5])
s

0    1
1    2
2    3
3    4
4    5
dtype: int64

roller=s.rolling(window=3)
roller

Rolling [window=3,center=False,axis=0]

roller.mean()

0    NaN
1    NaN
2    2.0
3    3.0
4    4.0
dtype: float64

roller.sum()

0     NaN
1     NaN
2     6.0
3     9.0
4    12.0
dtype: float64

s2 = pd.Series([1,2,6,16,30])
s2

0     1
1     2
2     6
3    16
4    30
dtype: int64

roller.cov(s2)

0     NaN
1     NaN
2     2.5
3     7.0
4    12.0
dtype: float64

roller.corr(s2)

0         NaN
1         NaN
2    0.944911
3    0.970725
4    0.995402
dtype: float64

# 此外，还支持使用 apply 传入自定义函数，其传入值是对应窗口的 Series ，例如上述的均值函数可以等效表示：
roller.apply(lambda x:x.mean())

0    NaN
1    NaN
2    2.0
3    3.0
4    4.0
dtype: float64

'''shift, diff, pct_change 是一组类滑窗函数，它们的公共参数为 periods=n ，默认为 1，分别表示取向前第 n
个元素的值、与向前第 n 个元素做差（与 Numpy 中不同，后者表示 n 阶差分）、与向前第 n 个元素相比计
算增长率。这里的 n 可以为负，表示反方向的类似操作。
'''
s=pd.Series([1,3,6,10,15])
s.shift(2)

0    NaN
1    NaN
2    1.0
3    3.0
4    6.0
dtype: float64

s.diff(3)

0     NaN
1     NaN
2     NaN
3     9.0
4    12.0
dtype: float64

s.pct_change() 
#表示当前元素与先前元素的相差百分比，当然指定periods=n,表示当前元素与先前n 个元素的相差百分比。

0         NaN
1    2.000000
2    1.000000
3    0.666667
4    0.500000
dtype: float64

s.shift(-1)

0     3.0
1     6.0
2    10.0
3    15.0
4     NaN
dtype: float64

s.diff(-2)

0   -5.0
1   -7.0
2   -9.0
3    NaN
4    NaN
dtype: float64

# 将其视作类滑窗函数的原因是，它们的功能可以用窗口大小为 n+1 的 rolling 方法等价代替：
s.rolling(3).apply(lambda x:list(x)[0]) # s.shift(2)

0    NaN
1    NaN
2    1.0
3    3.0
4    6.0
dtype: float64

s.rolling(4).apply(lambda x:list(x)[-1]-list(x)[0]) # s.diff(3)

0     NaN
1     NaN
2     NaN
3     9.0
4    12.0
dtype: float64

def my_pct(x):
    L=list(x)
    return L[-1]/L[0]-1

s.rolling(2).apply(my_pct)

0         NaN
1    2.000000
2    1.000000
3    0.666667
4    0.500000
dtype: float64

2.4.2 扩张窗口

'''扩张窗口又称累计窗口，可以理解为一个动态长度的窗口，其窗口的大小就是从序列开始处到具体操作的对
应位置，其使用的聚合函数会作用于这些逐步扩张的窗口上。具体地说，设序列为 a1, a2, a3, a4，则其每个
位置对应的窗口即 [a1]、[a1, a2]、[a1, a2, a3]、[a1, a2, a3, a4]。'''
s=pd.Series([1,3,6,10])
s

0     1
1     3
2     6
3    10
4    15
dtype: int64

s.expanding().mean()

0    1.000000
1    2.000000
2    3.333333
3    5.000000
4    7.000000
dtype: float64

2.5 练习

# 2.5.1 Ex1：口袋妖怪数据集
df = pd.read_csv('E:/DW学习/Python 学习代码/joyful-pandas-master/data/pokemon.csv')
df

	#	Name	Type 1	Type 2	Total	HP	Attack	Defense	Sp. Atk	Sp. Def	Speed
0	1	Bulbasaur	Grass	Poison	318	45	49	49	65	65	45
1	2	Ivysaur	Grass	Poison	405	60	62	63	80	80	60
2	3	Venusaur	Grass	Poison	525	80	82	83	100	100	80
3	3	VenusaurMega Venusaur	Grass	Poison	625	80	100	123	122	120	80
4	4	Charmander	Fire	NaN	309	39	52	43	60	50	65
...	...	...	...	...	...	...	...	...	...	...	...
795	719	Diancie	Rock	Fairy	600	50	100	150	100	150	50
796	719	DiancieMega Diancie	Rock	Fairy	700	50	160	110	160	110	110
797	720	HoopaHoopa Confined	Psychic	Ghost	600	80	110	60	150	130	70
798	720	HoopaHoopa Unbound	Psychic	Dark	680	80	160	60	170	130	80
799	721	Volcanion	Fire	Water	600	80	110	120	130	90	70

800 rows × 11 columns

# 1. 对 HP, Attack, Defense, Sp. Atk, Sp. Def, Speed 进行加总，验证是否为 Total 值。
(df[df.columns[5:]].sum(1)==df['Total']).all()

True

# 2. 对于 # 重复的妖怪只保留第一条记录，解决以下问题：
# (a) 求第一属性的种类数量和前三多数量对应的种类
# 备注:如果想要观察多个列组合的唯一值，可以使用 drop_duplicates 
df_temp=df.drop_duplicates('#')
df_temp.head()

	#	Name	Type 1	Type 2	Total	HP	Attack	Defense	Sp. Atk	Sp. Def	Speed
0	1	Bulbasaur	Grass	Poison	318	45	49	49	65	65	45
1	2	Ivysaur	Grass	Poison	405	60	62	63	80	80	60
2	3	Venusaur	Grass	Poison	525	80	82	83	100	100	80
4	4	Charmander	Fire	NaN	309	39	52	43	60	50	65
5	5	Charmeleon	Fire	NaN	405	58	64	58	80	65	80

df_temp['Type 1'].nunique()

18

df_temp['Type 1'].value_counts()

Water       105
Normal       93
Grass        66
Bug          63
Fire         47
Psychic      47
Rock         41
Electric     36
Ground       30
Dark         28
Poison       28
Fighting     25
Dragon       24
Ice          23
Ghost        23
Steel        22
Fairy        17
Flying        3
Name: Type 1, dtype: int64

df_temp['Type 1'].value_counts().index[:3]

Index(['Water', 'Normal', 'Grass'], dtype='object')

# (b) 求第一属性和第二属性的组合种类
df_temp.drop_duplicates(['Type 1','Type 2'])

	#	Name	Type 1	Type 2	Total	HP	Attack	Defense	Sp. Atk	Sp. Def	Speed
0	1	Bulbasaur	Grass	Poison	318	45	49	49	65	65	45
4	4	Charmander	Fire	NaN	309	39	52	43	60	50	65
6	6	Charizard	Fire	Flying	534	78	84	78	109	85	100
9	7	Squirtle	Water	NaN	314	44	48	65	50	64	43
13	10	Caterpie	Bug	NaN	195	45	30	35	20	20	45
...	...	...	...	...	...	...	...	...	...	...	...
773	703	Carbink	Rock	Fairy	500	50	50	150	50	150	50
778	708	Phantump	Ghost	Grass	309	43	70	48	50	60	38
790	714	Noibat	Flying	Dragon	245	40	30	35	45	40	55
797	720	HoopaHoopa Confined	Psychic	Ghost	600	80	110	60	150	130	70
799	721	Volcanion	Fire	Water	600	80	110	120	130	90	70

143 rows × 11 columns

df_temp.drop_duplicates(['Type 1','Type 2']).shape[0]

143

# (c) 求尚未出现过的属性组合
L_all=[i+' '+j for i in df['Type 1'].unique() for j in (df['Type 1'].unique().tolist()+[''])]
L_all

['Grass Grass',
 'Grass Fire',
 'Grass Water',
 'Grass Bug',
 'Grass Normal',
 'Grass Poison',
 'Grass Electric',
 'Grass Ground',
 'Grass Fairy',
 'Grass Fighting',
 'Grass Psychic',
 'Grass Rock',
 'Grass Ghost',
 'Grass Ice',
 'Grass Dragon',
 'Grass Dark',
 'Grass Steel',
 'Grass Flying',
 'Grass ',
 'Fire Grass',
 'Fire Fire',
 'Fire Water',
 'Fire Bug',
 'Fire Normal',
 'Fire Poison',
 'Fire Electric',
 'Fire Ground',
 'Fire Fairy',
 'Fire Fighting',
 'Fire Psychic',
 'Fire Rock',
 'Fire Ghost',
 'Fire Ice',
 'Fire Dragon',
 'Fire Dark',
 'Fire Steel',
 'Fire Flying',
 'Fire ',
 'Water Grass',
 'Water Fire',
 'Water Water',
 'Water Bug',
 'Water Normal',
 'Water Poison',
 'Water Electric',
 'Water Ground',
 'Water Fairy',
 'Water Fighting',
 'Water Psychic',
 'Water Rock',
 'Water Ghost',
 'Water Ice',
 'Water Dragon',
 'Water Dark',
 'Water Steel',
 'Water Flying',
 'Water ',
 'Bug Grass',
 'Bug Fire',
 'Bug Water',
 'Bug Bug',
 'Bug Normal',
 'Bug Poison',
 'Bug Electric',
 'Bug Ground',
 'Bug Fairy',
 'Bug Fighting',
 'Bug Psychic',
 'Bug Rock',
 'Bug Ghost',
 'Bug Ice',
 'Bug Dragon',
 'Bug Dark',
 'Bug Steel',
 'Bug Flying',
 'Bug ',
 'Normal Grass',
 'Normal Fire',
 'Normal Water',
 'Normal Bug',
 'Normal Normal',
 'Normal Poison',
 'Normal Electric',
 'Normal Ground',
 'Normal Fairy',
 'Normal Fighting',
 'Normal Psychic',
 'Normal Rock',
 'Normal Ghost',
 'Normal Ice',
 'Normal Dragon',
 'Normal Dark',
 'Normal Steel',
 'Normal Flying',
 'Normal ',
 'Poison Grass',
 'Poison Fire',
 'Poison Water',
 'Poison Bug',
 'Poison Normal',
 'Poison Poison',
 'Poison Electric',
 'Poison Ground',
 'Poison Fairy',
 'Poison Fighting',
 'Poison Psychic',
 'Poison Rock',
 'Poison Ghost',
 'Poison Ice',
 'Poison Dragon',
 'Poison Dark',
 'Poison Steel',
 'Poison Flying',
 'Poison ',
 'Electric Grass',
 'Electric Fire',
 'Electric Water',
 'Electric Bug',
 'Electric Normal',
 'Electric Poison',
 'Electric Electric',
 'Electric Ground',
 'Electric Fairy',
 'Electric Fighting',
 'Electric Psychic',
 'Electric Rock',
 'Electric Ghost',
 'Electric Ice',
 'Electric Dragon',
 'Electric Dark',
 'Electric Steel',
 'Electric Flying',
 'Electric ',
 'Ground Grass',
 'Ground Fire',
 'Ground Water',
 'Ground Bug',
 'Ground Normal',
 'Ground Poison',
 'Ground Electric',
 'Ground Ground',
 'Ground Fairy',
 'Ground Fighting',
 'Ground Psychic',
 'Ground Rock',
 'Ground Ghost',
 'Ground Ice',
 'Ground Dragon',
 'Ground Dark',
 'Ground Steel',
 'Ground Flying',
 'Ground ',
 'Fairy Grass',
 'Fairy Fire',
 'Fairy Water',
 'Fairy Bug',
 'Fairy Normal',
 'Fairy Poison',
 'Fairy Electric',
 'Fairy Ground',
 'Fairy Fairy',
 'Fairy Fighting',
 'Fairy Psychic',
 'Fairy Rock',
 'Fairy Ghost',
 'Fairy Ice',
 'Fairy Dragon',
 'Fairy Dark',
 'Fairy Steel',
 'Fairy Flying',
 'Fairy ',
 'Fighting Grass',
 'Fighting Fire',
 'Fighting Water',
 'Fighting Bug',
 'Fighting Normal',
 'Fighting Poison',
 'Fighting Electric',
 'Fighting Ground',
 'Fighting Fairy',
 'Fighting Fighting',
 'Fighting Psychic',
 'Fighting Rock',
 'Fighting Ghost',
 'Fighting Ice',
 'Fighting Dragon',
 'Fighting Dark',
 'Fighting Steel',
 'Fighting Flying',
 'Fighting ',
 'Psychic Grass',
 'Psychic Fire',
 'Psychic Water',
 'Psychic Bug',
 'Psychic Normal',
 'Psychic Poison',
 'Psychic Electric',
 'Psychic Ground',
 'Psychic Fairy',
 'Psychic Fighting',
 'Psychic Psychic',
 'Psychic Rock',
 'Psychic Ghost',
 'Psychic Ice',
 'Psychic Dragon',
 'Psychic Dark',
 'Psychic Steel',
 'Psychic Flying',
 'Psychic ',
 'Rock Grass',
 'Rock Fire',
 'Rock Water',
 'Rock Bug',
 'Rock Normal',
 'Rock Poison',
 'Rock Electric',
 'Rock Ground',
 'Rock Fairy',
 'Rock Fighting',
 'Rock Psychic',
 'Rock Rock',
 'Rock Ghost',
 'Rock Ice',
 'Rock Dragon',
 'Rock Dark',
 'Rock Steel',
 'Rock Flying',
 'Rock ',
 'Ghost Grass',
 'Ghost Fire',
 'Ghost Water',
 'Ghost Bug',
 'Ghost Normal',
 'Ghost Poison',
 'Ghost Electric',
 'Ghost Ground',
 'Ghost Fairy',
 'Ghost Fighting',
 'Ghost Psychic',
 'Ghost Rock',
 'Ghost Ghost',
 'Ghost Ice',
 'Ghost Dragon',
 'Ghost Dark',
 'Ghost Steel',
 'Ghost Flying',
 'Ghost ',
 'Ice Grass',
 'Ice Fire',
 'Ice Water',
 'Ice Bug',
 'Ice Normal',
 'Ice Poison',
 'Ice Electric',
 'Ice Ground',
 'Ice Fairy',
 'Ice Fighting',
 'Ice Psychic',
 'Ice Rock',
 'Ice Ghost',
 'Ice Ice',
 'Ice Dragon',
 'Ice Dark',
 'Ice Steel',
 'Ice Flying',
 'Ice ',
 'Dragon Grass',
 'Dragon Fire',
 'Dragon Water',
 'Dragon Bug',
 'Dragon Normal',
 'Dragon Poison',
 'Dragon Electric',
 'Dragon Ground',
 'Dragon Fairy',
 'Dragon Fighting',
 'Dragon Psychic',
 'Dragon Rock',
 'Dragon Ghost',
 'Dragon Ice',
 'Dragon Dragon',
 'Dragon Dark',
 'Dragon Steel',
 'Dragon Flying',
 'Dragon ',
 'Dark Grass',
 'Dark Fire',
 'Dark Water',
 'Dark Bug',
 'Dark Normal',
 'Dark Poison',
 'Dark Electric',
 'Dark Ground',
 'Dark Fairy',
 'Dark Fighting',
 'Dark Psychic',
 'Dark Rock',
 'Dark Ghost',
 'Dark Ice',
 'Dark Dragon',
 'Dark Dark',
 'Dark Steel',
 'Dark Flying',
 'Dark ',
 'Steel Grass',
 'Steel Fire',
 'Steel Water',
 'Steel Bug',
 'Steel Normal',
 'Steel Poison',
 'Steel Electric',
 'Steel Ground',
 'Steel Fairy',
 'Steel Fighting',
 'Steel Psychic',
 'Steel Rock',
 'Steel Ghost',
 'Steel Ice',
 'Steel Dragon',
 'Steel Dark',
 'Steel Steel',
 'Steel Flying',
 'Steel ',
 'Flying Grass',
 'Flying Fire',
 'Flying Water',
 'Flying Bug',
 'Flying Normal',
 'Flying Poison',
 'Flying Electric',
 'Flying Ground',
 'Flying Fairy',
 'Flying Fighting',
 'Flying Psychic',
 'Flying Rock',
 'Flying Ghost',
 'Flying Ice',
 'Flying Dragon',
 'Flying Dark',
 'Flying Steel',
 'Flying Flying',
 'Flying ']

len(L_all)

342

df['Type 1']+' '+df['Type 2']

0       Grass Poison
1       Grass Poison
2       Grass Poison
3       Grass Poison
4                NaN
           ...      
795       Rock Fairy
796       Rock Fairy
797    Psychic Ghost
798     Psychic Dark
799       Fire Water
Length: 800, dtype: object

L_part=[i+' '+j for i,j in zip(df['Type 1'],df['Type 2'].replace(np.nan,''))]
L_part

['Grass Poison',
 'Grass Poison',
 'Grass Poison',
 'Grass Poison',
 'Fire ',
 'Fire ',
 'Fire Flying',
 'Fire Dragon',
 'Fire Flying',
 'Water ',
 'Water ',
 'Water ',
 'Water ',
 'Bug ',
 'Bug ',
 'Bug Flying',
 'Bug Poison',
 'Bug Poison',
 'Bug Poison',
 'Bug Poison',
 'Normal Flying',
 'Normal Flying',
 'Normal Flying',
 'Normal Flying',
 'Normal ',
 'Normal ',
 'Normal Flying',
 'Normal Flying',
 'Poison ',
 'Poison ',
 'Electric ',
 'Electric ',
 'Ground ',
 'Ground ',
 'Poison ',
 'Poison ',
 'Poison Ground',
 'Poison ',
 'Poison ',
 'Poison Ground',
 'Fairy ',
 'Fairy ',
 'Fire ',
 'Fire ',
 'Normal Fairy',
 'Normal Fairy',
 'Poison Flying',
 'Poison Flying',
 'Grass Poison',
 'Grass Poison',
 'Grass Poison',
 'Bug Grass',
 'Bug Grass',
 'Bug Poison',
 'Bug Poison',
 'Ground ',
 'Ground ',
 'Normal ',
 'Normal ',
 'Water ',
 'Water ',
 'Fighting ',
 'Fighting ',
 'Fire ',
 'Fire ',
 'Water ',
 'Water ',
 'Water Fighting',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Fighting ',
 'Fighting ',
 'Fighting ',
 'Grass Poison',
 'Grass Poison',
 'Grass Poison',
 'Water Poison',
 'Water Poison',
 'Rock Ground',
 'Rock Ground',
 'Rock Ground',
 'Fire ',
 'Fire ',
 'Water Psychic',
 'Water Psychic',
 'Water Psychic',
 'Electric Steel',
 'Electric Steel',
 'Normal Flying',
 'Normal Flying',
 'Normal Flying',
 'Water ',
 'Water Ice',
 'Poison ',
 'Poison ',
 'Water ',
 'Water Ice',
 'Ghost Poison',
 'Ghost Poison',
 'Ghost Poison',
 'Ghost Poison',
 'Rock Ground',
 'Psychic ',
 'Psychic ',
 'Water ',
 'Water ',
 'Electric ',
 'Electric ',
 'Grass Psychic',
 'Grass Psychic',
 'Ground ',
 'Ground ',
 'Fighting ',
 'Fighting ',
 'Normal ',
 'Poison ',
 'Poison ',
 'Ground Rock',
 'Ground Rock',
 'Normal ',
 'Grass ',
 'Normal ',
 'Normal ',
 'Water ',
 'Water ',
 'Water ',
 'Water ',
 'Water ',
 'Water Psychic',
 'Psychic Fairy',
 'Bug Flying',
 'Ice Psychic',
 'Electric ',
 'Fire ',
 'Bug ',
 'Bug Flying',
 'Normal ',
 'Water ',
 'Water Flying',
 'Water Dark',
 'Water Ice',
 'Normal ',
 'Normal ',
 'Water ',
 'Electric ',
 'Fire ',
 'Normal ',
 'Rock Water',
 'Rock Water',
 'Rock Water',
 'Rock Water',
 'Rock Flying',
 'Rock Flying',
 'Normal ',
 'Ice Flying',
 'Electric Flying',
 'Fire Flying',
 'Dragon ',
 'Dragon ',
 'Dragon Flying',
 'Psychic ',
 'Psychic Fighting',
 'Psychic ',
 'Psychic ',
 'Grass ',
 'Grass ',
 'Grass ',
 'Fire ',
 'Fire ',
 'Fire ',
 'Water ',
 'Water ',
 'Water ',
 'Normal ',
 'Normal ',
 'Normal Flying',
 'Normal Flying',
 'Bug Flying',
 'Bug Flying',
 'Bug Poison',
 'Bug Poison',
 'Poison Flying',
 'Water Electric',
 'Water Electric',
 'Electric ',
 'Fairy ',
 'Normal Fairy',
 'Fairy ',
 'Fairy Flying',
 'Psychic Flying',
 'Psychic Flying',
 'Electric ',
 'Electric ',
 'Electric ',
 'Electric Dragon',
 'Grass ',
 'Water Fairy',
 'Water Fairy',
 'Rock ',
 'Water ',
 'Grass Flying',
 'Grass Flying',
 'Grass Flying',
 'Normal ',
 'Grass ',
 'Grass ',
 'Bug Flying',
 'Water Ground',
 'Water Ground',
 'Psychic ',
 'Dark ',
 'Dark Flying',
 'Water Psychic',
 'Ghost ',
 'Psychic ',
 'Psychic ',
 'Normal Psychic',
 'Bug ',
 'Bug Steel',
 'Normal ',
 'Ground Flying',
 'Steel Ground',
 'Steel Ground',
 'Fairy ',
 'Fairy ',
 'Water Poison',
 'Bug Steel',
 'Bug Steel',
 'Bug Rock',
 'Bug Fighting',
 'Bug Fighting',
 'Dark Ice',
 'Normal ',
 'Normal ',
 'Fire ',
 'Fire Rock',
 'Ice Ground',
 'Ice Ground',
 'Water Rock',
 'Water ',
 'Water ',
 'Ice Flying',
 'Water Flying',
 'Steel Flying',
 'Dark Fire',
 'Dark Fire',
 'Dark Fire',
 'Water Dragon',
 'Ground ',
 'Ground ',
 'Normal ',
 'Normal ',
 'Normal ',
 'Fighting ',
 'Fighting ',
 'Ice Psychic',
 'Electric ',
 'Fire ',
 'Normal ',
 'Normal ',
 'Electric ',
 'Fire ',
 'Water ',
 'Rock Ground',
 'Rock Ground',
 'Rock Dark',
 'Rock Dark',
 'Psychic Flying',
 'Fire Flying',
 'Psychic Grass',
 'Grass ',
 'Grass ',
 'Grass ',
 'Grass Dragon',
 'Fire ',
 'Fire Fighting',
 'Fire Fighting',
 'Fire Fighting',
 'Water ',
 'Water Ground',
 'Water Ground',
 'Water Ground',
 'Dark ',
 'Dark ',
 'Normal ',
 'Normal ',
 'Bug ',
 'Bug ',
 'Bug Flying',
 'Bug ',
 'Bug Poison',
 'Water Grass',
 'Water Grass',
 'Water Grass',
 'Grass ',
 'Grass Dark',
 'Grass Dark',
 'Normal Flying',
 'Normal Flying',
 'Water Flying',
 'Water Flying',
 'Psychic Fairy',
 'Psychic Fairy',
 'Psychic Fairy',
 'Psychic Fairy',
 'Bug Water',
 'Bug Flying',
 'Grass ',
 'Grass Fighting',
 'Normal ',
 'Normal ',
 'Normal ',
 'Bug Ground',
 'Bug Flying',
 'Bug Ghost',
 'Normal ',
 'Normal ',
 'Normal ',
 'Fighting ',
 'Fighting ',
 'Normal Fairy',
 'Rock ',
 'Normal ',
 'Normal ',
 'Dark Ghost',
 'Dark Ghost',
 'Steel Fairy',
 'Steel Fairy',
 'Steel Rock',
 'Steel Rock',
 'Steel Rock',
 'Steel ',
 'Fighting Psychic',
 'Fighting Psychic',
 'Fighting Psychic',
 'Electric ',
 'Electric ',
 'Electric ',
 'Electric ',
 'Electric ',
 'Bug ',
 'Bug ',
 'Grass Poison',
 'Poison ',
 'Poison ',
 'Water Dark',
 'Water Dark',
 'Water Dark',
 'Water ',
 'Water ',
 'Fire Ground',
 'Fire Ground',
 'Fire Ground',
 'Fire ',
 'Psychic ',
 'Psychic ',
 'Normal ',
 'Ground ',
 'Ground Dragon',
 'Ground Dragon',
 'Grass ',
 'Grass Dark',
 'Normal Flying',
 'Dragon Flying',
 'Dragon Fairy',
 'Normal ',
 'Poison ',
 'Rock Psychic',
 'Rock Psychic',
 'Water Ground',
 'Water Ground',
 'Water ',
 'Water Dark',
 'Ground Psychic',
 'Ground Psychic',
 'Rock Grass',
 'Rock Grass',
 'Rock Bug',
 'Rock Bug',
 'Water ',
 'Water ',
 'Normal ',
 'Normal ',
 'Ghost ',
 'Ghost ',
 'Ghost ',
 'Ghost ',
 'Ghost ',
 'Grass Flying',
 'Psychic ',
 'Dark ',
 'Dark ',
 'Psychic ',
 'Ice ',
 'Ice ',
 'Ice ',
 'Ice Water',
 'Ice Water',
 'Ice Water',
 'Water ',
 'Water ',
 'Water ',
 'Water Rock',
 'Water ',
 'Dragon ',
 'Dragon ',
 'Dragon Flying',
 'Dragon Flying',
 'Steel Psychic',
 'Steel Psychic',
 'Steel Psychic',
 'Steel Psychic',
 'Rock ',
 'Ice ',
 'Steel ',
 'Dragon Psychic',
 'Dragon Psychic',
 'Dragon Psychic',
 'Dragon Psychic',
 'Water ',
 'Water ',
 'Ground ',
 'Ground Fire',
 'Dragon Flying',
 'Dragon Flying',
 'Steel Psychic',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Grass ',
 'Grass ',
 'Grass Ground',
 'Fire ',
 'Fire Fighting',
 'Fire Fighting',
 'Water ',
 'Water ',
 'Water Steel',
 'Normal Flying',
 'Normal Flying',
 'Normal Flying',
 'Normal ',
 'Normal Water',
 'Bug ',
 'Bug ',
 'Electric ',
 'Electric ',
 'Electric ',
 'Grass Poison',
 'Grass Poison',
 'Rock ',
 'Rock ',
 'Rock Steel',
 'Rock Steel',
 'Bug ',
 'Bug Grass',
 'Bug Ground',
 'Bug Steel',
 'Bug Flying',
 'Bug Flying',
 'Bug Flying',
 'Electric ',
 'Water ',
 'Water ',
 'Grass ',
 'Grass ',
 'Water ',
 'Water Ground',
 'Normal ',
 'Ghost Flying',
 'Ghost Flying',
 'Normal ',
 'Normal ',
 'Normal Fighting',
 'Ghost ',
 'Dark Flying',
 'Normal ',
 'Normal ',
 'Psychic ',
 'Poison Dark',
 'Poison Dark',
 'Steel Psychic',
 'Steel Psychic',
 'Rock ',
 'Psychic Fairy',
 'Normal ',
 'Normal Flying',
 'Ghost Dark',
 'Dragon Ground',
 'Dragon Ground',
 'Dragon Ground',
 'Dragon Ground',
 'Normal ',
 'Fighting ',
 'Fighting Steel',
 'Fighting Steel',
 'Ground ',
 'Ground ',
 'Poison Bug',
 'Poison Dark',
 'Poison Fighting',
 'Poison Fighting',
 'Grass ',
 'Water ',
 'Water ',
 'Water Flying',
 'Grass Ice',
 'Grass Ice',
 'Grass Ice',
 'Dark Ice',
 'Electric Steel',
 'Normal ',
 'Ground Rock',
 'Grass ',
 'Electric ',
 'Fire ',
 'Fairy Flying',
 'Bug Flying',
 'Grass ',
 'Ice ',
 'Ground Flying',
 'Ice Ground',
 'Normal ',
 'Psychic Fighting',
 'Psychic Fighting',
 'Rock Steel',
 'Ghost ',
 'Ice Ghost',
 'Electric Ghost',
 'Electric Fire',
 'Electric Water',
 'Electric Ice',
 'Electric Flying',
 'Electric Grass',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Steel Dragon',
 'Water Dragon',
 'Fire Steel',
 'Normal ',
 'Ghost Dragon',
 'Ghost Dragon',
 'Psychic ',
 'Water ',
 'Water ',
 'Dark ',
 'Grass ',
 'Grass Flying',
 'Normal ',
 'Psychic Fire',
 'Grass ',
 'Grass ',
 'Grass ',
 'Fire ',
 'Fire Fighting',
 'Fire Fighting',
 'Water ',
 'Water ',
 'Water ',
 'Normal ',
 'Normal ',
 'Normal ',
 'Normal ',
 'Normal ',
 'Dark ',
 'Dark ',
 'Grass ',
 'Grass ',
 'Fire ',
 'Fire ',
 'Water ',
 'Water ',
 'Psychic ',
 'Psychic ',
 'Normal Flying',
 'Normal Flying',
 'Normal Flying',
 'Electric ',
 'Electric ',
 'Rock ',
 'Rock ',
 'Rock ',
 'Psychic Flying',
 'Psychic Flying',
 'Ground ',
 'Ground Steel',
 'Normal ',
 'Normal Fairy',
 'Fighting ',
 'Fighting ',
 'Fighting ',
 'Water ',
 'Water Ground',
 'Water Ground',
 'Fighting ',
 'Fighting ',
 'Bug Grass',
 'Bug Grass',
 'Bug Grass',
 'Bug Poison',
 'Bug Poison',
 'Bug Poison',
 'Grass Fairy',
 'Grass Fairy',
 'Grass ',
 'Grass ',
 'Water ',
 'Ground Dark',
 'Ground Dark',
 'Ground Dark',
 'Fire ',
 'Fire ',
 'Fire Psychic',
 'Grass ',
 'Bug Rock',
 'Bug Rock',
 'Dark Fighting',
 'Dark Fighting',
 'Psychic Flying',
 'Ghost ',
 'Ghost ',
 'Water Rock',
 'Water Rock',
 'Rock Flying',
 'Rock Flying',
 'Poison ',
 'Poison ',
 'Dark ',
 'Dark ',
 'Normal ',
 'Normal ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Water Flying',
 'Water Flying',
 'Ice ',
 'Ice ',
 'Ice ',
 'Normal Grass',
 'Normal Grass',
 'Electric Flying',
 'Bug ',
 'Bug Steel',
 'Grass Poison',
 'Grass Poison',
 'Water Ghost',
 'Water Ghost',
 'Water ',
 'Bug Electric',
 'Bug Electric',
 'Grass Steel',
 'Grass Steel',
 'Steel ',
 'Steel ',
 'Steel ',
 'Electric ',
 'Electric ',
 'Electric ',
 'Psychic ',
 'Psychic ',
 'Ghost Fire',
 'Ghost Fire',
 'Ghost Fire',
 'Dragon ',
 'Dragon ',
 'Dragon ',
 'Ice ',
 'Ice ',
 'Ice ',
 'Bug ',
 'Bug ',
 'Ground Electric',
 'Fighting ',
 'Fighting ',
 'Dragon ',
 'Ground Ghost',
 'Ground Ghost',
 'Dark Steel',
 'Dark Steel',
 'Normal ',
 'Normal Flying',
 'Normal Flying',
 'Dark Flying',
 'Dark Flying',
 'Fire ',
 'Bug Steel',
 'Dark Dragon',
 'Dark Dragon',
 'Dark Dragon',
 'Bug Fire',
 'Bug Fire',
 'Steel Fighting',
 'Rock Fighting',
 'Grass Fighting',
 'Flying ',
 'Flying ',
 'Electric Flying',
 'Electric Flying',
 'Dragon Fire',
 'Dragon Electric',
 'Ground Flying',
 'Ground Flying',
 'Dragon Ice',
 'Dragon Ice',
 'Dragon Ice',
 'Water Fighting',
 'Water Fighting',
 'Normal Psychic',
 'Normal Fighting',
 'Bug Steel',
 'Grass ',
 'Grass ',
 'Grass Fighting',
 'Fire ',
 'Fire ',
 'Fire Psychic',
 'Water ',
 'Water ',
 'Water Dark',
 'Normal ',
 'Normal Ground',
 'Normal Flying',
 'Fire Flying',
 'Fire Flying',
 'Bug ',
 'Bug ',
 'Bug Flying',
 'Fire Normal',
 'Fire Normal',
 'Fairy ',
 'Fairy ',
 'Fairy ',
 'Grass ',
 'Grass ',
 'Fighting ',
 'Fighting Dark',
 'Normal ',
 'Psychic ',
 'Psychic ',
 'Psychic ',
 'Steel Ghost',
 'Steel Ghost',
 'Steel Ghost',
 'Steel Ghost',
 'Fairy ',
 'Fairy ',
 'Fairy ',
 'Fairy ',
 'Dark Psychic',
 'Dark Psychic',
 'Rock Water',
 'Rock Water',
 'Poison Water',
 'Poison Dragon',
 'Water ',
 'Water ',
 'Electric Normal',
 'Electric Normal',
 'Rock Dragon',
 'Rock Dragon',
 'Rock Ice',
 'Rock Ice',
 'Fairy ',
 'Fighting Flying',
 'Electric Fairy',
 'Rock Fairy',
 'Dragon ',
 'Dragon ',
 'Dragon ',
 'Steel Fairy',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ghost Grass',
 'Ice ',
 'Ice ',
 'Flying Dragon',
 'Flying Dragon',
 'Fairy ',
 'Dark Flying',
 'Dragon Ground',
 'Rock Fairy',
 'Rock Fairy',
 'Psychic Ghost',
 'Psychic Dark',
 'Fire Water']

set(L_all).difference(L_part)

{'Bug Bug',
 'Bug Dark',
 'Bug Dragon',
 'Bug Fairy',
 'Bug Ice',
 'Bug Normal',
 'Bug Psychic',
 'Dark Bug',
 'Dark Dark',
 'Dark Electric',
 'Dark Fairy',
 'Dark Grass',
 'Dark Ground',
 'Dark Normal',
 'Dark Poison',
 'Dark Rock',
 'Dark Water',
 'Dragon Bug',
 'Dragon Dark',
 'Dragon Dragon',
 'Dragon Fighting',
 'Dragon Ghost',
 'Dragon Grass',
 'Dragon Normal',
 'Dragon Poison',
 'Dragon Rock',
 'Dragon Steel',
 'Dragon Water',
 'Electric Bug',
 'Electric Dark',
 'Electric Electric',
 'Electric Fighting',
 'Electric Ground',
 'Electric Poison',
 'Electric Psychic',
 'Electric Rock',
 'Fairy Bug',
 'Fairy Dark',
 'Fairy Dragon',
 'Fairy Electric',
 'Fairy Fairy',
 'Fairy Fighting',
 'Fairy Fire',
 'Fairy Ghost',
 'Fairy Grass',
 'Fairy Ground',
 'Fairy Ice',
 'Fairy Normal',
 'Fairy Poison',
 'Fairy Psychic',
 'Fairy Rock',
 'Fairy Steel',
 'Fairy Water',
 'Fighting Bug',
 'Fighting Dragon',
 'Fighting Electric',
 'Fighting Fairy',
 'Fighting Fighting',
 'Fighting Fire',
 'Fighting Ghost',
 'Fighting Grass',
 'Fighting Ground',
 'Fighting Ice',
 'Fighting Normal',
 'Fighting Poison',
 'Fighting Rock',
 'Fighting Water',
 'Fire Bug',
 'Fire Dark',
 'Fire Electric',
 'Fire Fairy',
 'Fire Fire',
 'Fire Ghost',
 'Fire Grass',
 'Fire Ice',
 'Fire Poison',
 'Flying Bug',
 'Flying Dark',
 'Flying Electric',
 'Flying Fairy',
 'Flying Fighting',
 'Flying Fire',
 'Flying Flying',
 'Flying Ghost',
 'Flying Grass',
 'Flying Ground',
 'Flying Ice',
 'Flying Normal',
 'Flying Poison',
 'Flying Psychic',
 'Flying Rock',
 'Flying Steel',
 'Flying Water',
 'Ghost Bug',
 'Ghost Electric',
 'Ghost Fairy',
 'Ghost Fighting',
 'Ghost Ghost',
 'Ghost Ground',
 'Ghost Ice',
 'Ghost Normal',
 'Ghost Psychic',
 'Ghost Rock',
 'Ghost Steel',
 'Ghost Water',
 'Grass Bug',
 'Grass Electric',
 'Grass Fire',
 'Grass Ghost',
 'Grass Grass',
 'Grass Normal',
 'Grass Rock',
 'Grass Water',
 'Ground Bug',
 'Ground Fairy',
 'Ground Fighting',
 'Ground Grass',
 'Ground Ground',
 'Ground Ice',
 'Ground Normal',
 'Ground Poison',
 'Ground Water',
 'Ice Bug',
 'Ice Dark',
 'Ice Dragon',
 'Ice Electric',
 'Ice Fairy',
 'Ice Fighting',
 'Ice Fire',
 'Ice Grass',
 'Ice Ice',
 'Ice Normal',
 'Ice Poison',
 'Ice Rock',
 'Ice Steel',
 'Normal Bug',
 'Normal Dark',
 'Normal Dragon',
 'Normal Electric',
 'Normal Fire',
 'Normal Ghost',
 'Normal Ice',
 'Normal Normal',
 'Normal Poison',
 'Normal Rock',
 'Normal Steel',
 'Poison Electric',
 'Poison Fairy',
 'Poison Fire',
 'Poison Ghost',
 'Poison Grass',
 'Poison Ice',
 'Poison Normal',
 'Poison Poison',
 'Poison Psychic',
 'Poison Rock',
 'Poison Steel',
 'Psychic Bug',
 'Psychic Dragon',
 'Psychic Electric',
 'Psychic Ground',
 'Psychic Ice',
 'Psychic Normal',
 'Psychic Poison',
 'Psychic Psychic',
 'Psychic Rock',
 'Psychic Steel',
 'Psychic Water',
 'Rock Electric',
 'Rock Fire',
 'Rock Ghost',
 'Rock Normal',
 'Rock Poison',
 'Rock Rock',
 'Steel Bug',
 'Steel Dark',
 'Steel Electric',
 'Steel Fire',
 'Steel Grass',
 'Steel Ice',
 'Steel Normal',
 'Steel Poison',
 'Steel Steel',
 'Steel Water',
 'Water Bug',
 'Water Fire',
 'Water Normal',
 'Water Water'}

len(set(L_all).difference(L_part))

188

# 3. 按照下述要求，构造 Series ：
# (a) 取出物攻，超过 120 的替换为 high ，不足 50 的替换为 low ，否则设为 mid
# 逻辑替换包括了 where 和 mask ，这两个函数是完全对称的：where 函数在传入条件为 False 的对应行进行
# 替换，而 mask 在传入条件为 True 的对应行进行替换，当不指定替换值时，替换为缺失
df['Attack'].mask(df['Attack']>120,'Hight').mask(df['Attack']<50,'Low').mask((df['Attack']<=120)&(df['Attack']>=50),'Mid')

0        Low
1        Mid
2        Mid
3        Mid
4        Mid
       ...  
795      Mid
796    Hight
797      Mid
798    Hight
799      Mid
Name: Attack, Length: 800, dtype: object

# (b) 取出第一属性，分别用 replace 和 apply 替换所有字母为大写
L=df['Type 1'].unique().tolist()
L

['Grass',
 'Fire',
 'Water',
 'Bug',
 'Normal',
 'Poison',
 'Electric',
 'Ground',
 'Fairy',
 'Fighting',
 'Psychic',
 'Rock',
 'Ghost',
 'Ice',
 'Dragon',
 'Dark',
 'Steel',
 'Flying']

df['Type 1'].replace(L,[str.upper(i) for i in L])

0        GRASS
1        GRASS
2        GRASS
3        GRASS
4         FIRE
        ...   
795       ROCK
796       ROCK
797    PSYCHIC
798    PSYCHIC
799       FIRE
Name: Type 1, Length: 800, dtype: object

df['Type 1'].apply(lambda x:str.upper(x))

0        GRASS
1        GRASS
2        GRASS
3        GRASS
4         FIRE
        ...   
795       ROCK
796       ROCK
797    PSYCHIC
798    PSYCHIC
799       FIRE
Name: Type 1, Length: 800, dtype: object

# (c) 求每个妖怪六项能力的离差，即所有能力中偏离中位数最大的值，添加到 df 并从大到小排序

df['Deviation']=df[df.columns[5:]].apply(lambda x:(x-x.median()).max(),1)
df['Deviation']

0      16.0
1      18.0
2      18.0
3      23.0
4      15.0
       ... 
795    50.0
796    50.0
797    70.0
798    90.0
799    40.0
Name: Deviation, Length: 800, dtype: float64

df.sort_values('Deviation',ascending=False)

	#	Name	Type 1	Type 2	Total	HP	Attack	Defense	Sp. Atk	Sp. Def	Speed	Deviation
230	213	Shuckle	Bug	Rock	505	20	10	230	10	230	5	210.0
121	113	Chansey	Normal	NaN	450	250	5	5	35	105	50	200.0
261	242	Blissey	Normal	NaN	540	255	10	10	75	135	55	180.0
333	306	AggronMega Aggron	Steel	NaN	630	70	140	230	60	80	50	150.0
224	208	SteelixMega Steelix	Steel	Ground	610	75	125	230	55	95	30	135.0
...	...	...	...	...	...	...	...	...	...	...	...	...
553	494	Victini	Psychic	Fire	600	100	100	100	100	100	100	0.0
206	191	Sunkern	Grass	NaN	180	30	30	30	30	30	30	0.0
255	236	Tyrogue	Fighting	NaN	210	35	35	35	35	35	35	0.0
427	385	Jirachi	Steel	Psychic	600	100	100	100	100	100	100	0.0
538	481	Mesprit	Psychic	NaN	580	80	105	105	105	105	80	0.0

800 rows × 12 columns