2012联邦选举委员会数据库
!git clone https://github.com/wesm/pydata-book
0 导入相关库
# 基础
import numpy as np # 处理数组
import pandas as pd # 读取数据&&DataFrame
import matplotlib.pyplot as plt # 制图
import seaborn as sns
from matplotlib import rcParams # 定义参数
from matplotlib.cm import rainbow # 配置颜色
%matplotlib inline
import warnings
warnings.filterwarnings('ignore') # 忽略警告信息
np.set_printoptions(precision=4) # 小数点后
pd.options.display.max_rows = 10 # 最大行数
1 读取文本文件(CSV格式)
!ls pydata-book/datasets/fec
fec = pd.read_csv('pydata-book/datasets/fec/P00000001-ALL.csv')
fec.info()
fec
fec.iloc[123456]
2 数据预处理
查看候选人名单并统计数量
unique_cands = fec.cand_nm.unique()
unique_cands
fec['cand_nm'].value_counts()
fec['cand_nm'].value_counts()[:5].plot.barh()
加入党派信息(Republican&Democrat)
parties = {'Bachmann, Michelle': 'Republican',
'Cain, Herman': 'Republican',
'Gingrich, Newt': 'Republican',
'Huntsman, Jon': 'Republican',
'Johnson, Gary Earl': 'Republican',
'McCotter, Thaddeus G': 'Republican',
'Obama, Barack': 'Democrat',
'Paul, Ron': 'Republican',
'Pawlenty, Timothy': 'Republican',
'Perry, Rick': 'Republican',
"Roemer, Charles E. 'Buddy' III": 'Republican',
'Romney, Mitt': 'Republican',
'Santorum, Rick': 'Republican'}
fec.cand_nm[123456:123460]
fec.cand_nm[123456:123460].map(parties)
fec['party'] = fec.cand_nm.map(parties)
fec['party'].value_counts()
查看各党派数量
fec['party'].value_counts()[::-1].plot.barh()
第一、包括赞助款&退款(负的出资额)
fec.contb_receipt_amt.value_counts()
(fec.contb_receipt_amt > 0).value_counts()
为了简化分析过程,限定该数据集只有正当出资额
fec = fec[fec.contb_receipt_amt > 0]
Obama, Barack
&Paul, Ron
最主要的两名候选人
fec_mrbo = fec[fec.cand_nm.isin(['Obama, Barack', 'Paul, Ron'])]
fec_mrbo
The kernel appears to have died. It will restart automatically.
按照书上所述
fec_mrbo = fec[fec.cand_nm.isin([‘Obama, Barack’, ‘Romney, Mitt’])]
3 根据职业和雇主统计赞助信息
fec.contbr_occupation.value_counts()[:10]
许多职业都涉及相同的基本工作类型(同一样东西有多重变体)
清理数据(将一个职业信息映射到另一个)
dict.get # 允许没有映射关系的职业也能’通过’
occ_mapping = {
'INFORMATION REQUESTED PER BEST EFFORTS' : 'NOT PROVIDED',
'INFORMATION REQUESTED' : 'NOT PROVIDED',
'INFORMATION REQUESTED (BEST EFFORTS)' : 'NOT PROVIDED',
'C.E.O.': 'CEO'
}
f = lambda x: occ_mapping.get(x, x)
fec.contbr_occupation = fec.contbr_occupation.map(f)
fec.contbr_occupation.value_counts()[:10].plot.barh()
对雇主信息进行同样处理
emp_mapping = {
'INFORMATION REQUESTED PER BEST EFFORTS' : 'NOT PROVIDED',
'INFORMATION REQUESTED' : 'NOT PROVIDED',
'SELF' : 'SELF-EMPLOYED',
'SELF EMPLOYED' : 'SELF-EMPLOYED',
}
f = lambda x: emp_mapping.get(x, x)
fec.contbr_employer = fec.contbr_employer.map(f)
fec.contbr_employer.value_counts()[:10].plot.barh()
通过
pd.pivot_table
根据党派和职业进行数据进行聚合
过滤掉总出资额不足200
万美元的数据
by_occupation = fec.pivot_table('contb_receipt_amt', index='contbr_occupation',
columns='party', aggfunc='sum')
over_2mm = by_occupation[by_occupation.sum(1) > 2000000]
over_2mm
over_2mm.plot(kind='barh', fontsize=14, grid=True)
def get_top_amounts(group, key, n=5):
totals = group.groupby(key)['contb_receipt_amt'].sum()
return totals.nlargest(n)
grouped = fec_mrbo.groupby('cand_nm')
grouped.apply(get_top_amounts, 'contbr_occupation', n=7)
grouped.apply(get_top_amounts, 'contbr_employer', n=10)
fec_mrbo_test = fec[fec.cand_nm.isin(['Obama, Barack', 'Paul, Ron'])]
fec_mrbo_test.shape
fec_mrbo_test
grouped = fec_mrbo_test[:500000].groupby('cand_nm')
grouped.apply(get_top_amounts, 'contbr_occupation', n=7)
grouped = fec_mrbo_test[:600000].groupby('cand_nm')
grouped.apply(get_top_amounts, 'contbr_occupation', n=7)
grouped = fec_mrbo_test[:700000].groupby('cand_nm')
grouped.apply(get_top_amounts, 'contbr_occupation', n=7)
grouped = fec_mrbo_test.groupby('cand_nm')
grouped.apply(get_top_amounts, 'contbr_occupation', n=7)
emmm
4 对出资额分组
利用
pd.cut函数
根据出资额的大小将数据离散化到多个面元中
bins = np.array([0, 1, 10, 100, 1000, 10000,
100000, 1000000, 10000000])
labels = pd.cut(fec_mrbo.contb_receipt_amt, bins)
labels
根据候选人名单以及面元标签对奥巴马和罗姆尼数据 & 绘制柱状图
grouped = fec_mrbo.groupby(['cand_nm', labels])
grouped.size()
grouped.size().unstack(0)
grouped.size().unstack(0).plot(kind=‘barh’)
小额赞助方面,Obama, Barack
获得的数量比Romney, Mitt
多得多
对出资额求和并在面元内规格化,便于图形化显示两位候选人各种赞助额的比例
bucket_sums = grouped.contb_receipt_amt.sum().unstack(0)
normed_sums = bucket_sums.div(bucket_sums.sum(axis=1), axis=0)
normed_sums
normed_sums[:-2].plot(kind='barh')
排除两个最大的面元(不是个人捐赠)
5 根据州统计赞助信息
根据候选人和州对数据进行聚合
grouped = fec_mrbo.groupby(['cand_nm', 'contbr_st'])
fec_mrbo.groupby(['cand_nm', 'contbr_st']).sum()
totals = grouped.contb_receipt_amt.sum().unstack(0).fillna(0)
totals = totals[totals.sum(1) > 100000]
totals[:10]
totals[:10].plot(kind='barh')
totals[:5].plot(kind='pie', subplots=True, legend=False, autopct='%.2f')
对各行除以总赞助额,就会得到各候选人在各州的总赞助额比例
percent = totals.div(totals.sum(1), axis=0)
percent[:10]
percent[:10].plot(kind='barh')
percent[:10].plot(kind='pie', subplots=True, legend=False, autopct='%.2f')