2021-07-13

Python学习笔记

数据分析之查看处理数据

// An highlighted block
txt[['Age','Sex']]
txt.loc[100:400]

txt[txt['Age']<40]

txt.loc[[100,110,120],["Age","Sex"]]

txt[["Age","Sex"]].loc[100:400]

txt.head(6)

txt[txt["Age"]<10].head(3)

排序

frame.sort_values(by=['a','b'],ascending = False)
frame.sort_values(by='c',ascending=True)
frame.sort_index(axis=1)
text['票价'].describe()

查看空值

df.isnull().sum()
df[['Age','Cabin']]

处理缺失值

df[df['Age']==None]=0
df[df['Age'].isnull()]=0
df[df['Age']==np.nan]=0

df.dropna().head()
df.fillna(0).head()

重复值

##观察重复值
df[df.duplicated()]
##处理重复值
df=df.drop_duplicates()
df.head()
df.to_csv('data.csv')

连续变量离散化

df['age_cut']=pd.cut(df['Age'],5,lables=[1,2,3,4,5])
df.head()

df['age_cut']=pd.cut(df['Age'],[0,5,15,30,50,80],lables=[1,2,3,4,5])
df['AgeBand'] = pd.qcut(df['Age'],[0,0.1,0.3,0.5,0.7,0.9],labels = [1,2,3,4,5])

一些分析

##
df['Sex'].value_counts()
df['Sex'].unique()
df['Sex'].nunique()
df['Sex_num'] = df['Sex'].replace(['male','female'],[1,2])
###从名字里提取出titile，也即Mr,Miss,Mrs
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)

数据重构

###concat
list_up = [text_left_up,text_right_up]
result_up = pd.concat(list_up,axis=1)
result_up.head()

##join&append
resul_up = text_left_up.join(text_right_up)
result_down = text_left_down.join(text_right_down)
result = result_up.append(result_down)

##merge&append
result_up = pd.merge(text_left_up,text_right_up,left_index=True,right_index=True)
result_down = pd.merge(text_left_down,text_right_down,left_index=True,right_index=True)
result = resul_up.append(result_down)

##dataframe to series
unit_result = text.stack().head()


##caculate
df = text['Fare'].groupby(text['Sex'])
df = text['Fare'].groupby(text['Sex']).sum()
df.mean()
##修改列名
text.groupby('Sex').agg({'Fare': 'mean', 'Pclass': 'count'}).rename(columns=
    {'Fare': 'mean_fare', 'Pclass': 'count_pclass'})

text.groupby(['A','B'])['Fare'].mean().head()


result = pd.merge(means,survived_sex,on='Sex')

#不同年龄的存活人数
survived_age = text['Survived'].groupby(text['Age']).sum()
survived_age.head()
#找出最大值的年龄段
survived_age[survived_age.values==survived_age.max()]
_sum = text['Survived'].sum()
print(_sum)
#首先计算总人数
_sum = text['Survived'].sum()

print("sum of person:"+str(_sum))

precetn =survived_age.max()/_sum

print("最大存活率："+str(precetn))

数据可视化

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

sex = text.groupby('Sex')['Survived'].sum()
sex.plot.bar()
plt.title('survived_count')
plt.show()

# 提示：计算男女中死亡人数 1表示生存，0表示死亡
text.groupby(['Sex','Survived'])['Survived'].count().unstack().plot(kind='bar',stacked='True')
plt.title('survived_count')
plt.ylabel('count')

# 计算不同票价中生存与死亡人数 1表示生存，0表示死亡
fare_sur = text.groupby(['Fare'])['Survived'].value_counts().sort_values(ascending=False)
fare_sur
# 排序后绘折线图
fig = plt.figure(figsize=(20, 18))
fare_sur.plot(grid=True)
plt.show()
# 排序前绘折线图
fare_sur1 = text.groupby(['Fare'])['Survived'].value_counts()
fare_sur1
fig = plt.figure(figsize=(20, 18))
fare_sur1.plot(grid=True)
plt.legend()
plt.show()



pclass_sur = text.groupby(['Pclass'])['Survived'].value_counts()
pclass_sur
import seaborn as sns
sns.countplot(x="Pclass", hue="Survived", data=text)

facet = sns.FacetGrid(text, hue="Survived",aspect=3)
facet.map(sns.kdeplot,'Age',shade= True)
facet.set(xlim=(0, text['Age'].max()))
facet.add_legend()


text.Age[text.Pclass == 1].plot(kind='kde')
text.Age[text.Pclass == 2].plot(kind='kde')
text.Age[text.Pclass == 3].plot(kind='kde')
plt.xlabel("age")
plt.legend((1,2,3),loc="best")

建模

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from IPython.display import Image

plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
plt.rcParams['figure.figsize'] = (10, 6)  # 设置输出图片大小

# 读取训练数集
train = pd.read_csv('train.csv')
train.shape

# 对分类变量进行填充
train['Cabin'] = train['Cabin'].fillna('NA')
train['Embarked'] = train['Embarked'].fillna('S')

# 对连续变量进行填充
train['Age'] = train['Age'].fillna(train['Age'].mean())

# 检查缺失值比例
train.isnull().sum().sort_values(ascending=False)

# 取出所有的输入特征
data = train[['Pclass','Sex','Age','SibSp','Parch','Fare', 'Embarked']]

# 进行虚拟变量转换
data = pd.get_dummies(data)

rom sklearn.model_selection import train_test_split
# 一般先取出X和y后再切割，有些情况会使用到未切割的，这时候X和y就可以用
X = data
y = train['Survived']
# 对数据集进行切割
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0)
# 查看数据形状
X_train.shape, X_test.shape

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier

# 默认参数逻辑回归模型
lr = LogisticRegression()
lr.fit(X_train, y_train)

# 查看训练集和测试集score值
print("Training set score: {:.2f}".format(lr.score(X_train, y_train)))
print("Testing set score: {:.2f}".format(lr.score(X_test, y_test)))

# 调整参数后的逻辑回归模型
lr2 = LogisticRegression(C=100)
lr2.fit(X_train, y_train)

print("Training set score: {:.2f}".format(lr2.score(X_train, y_train)))
print("Testing set score: {:.2f}".format(lr2.score(X_test, y_test)))