Python学习笔记
数据分析之查看处理数据
// An highlighted block
txt[ [ 'Age' , 'Sex' ] ]
txt. loc[ 100 : 400 ]
txt[ txt[ 'Age' ] < 40 ]
txt. loc[ [ 100 , 110 , 120 ] , [ "Age" , "Sex" ] ]
txt[ [ "Age" , "Sex" ] ] . loc[ 100 : 400 ]
txt. head( 6 )
txt[ txt[ "Age" ] < 10 ] . head( 3 )
排序
frame. sort_values( by= [ 'a' , 'b' ] , ascending = False )
frame. sort_values( by= 'c' , ascending= True )
frame. sort_index( axis= 1 )
text[ '票价' ] . describe( )
查看空值
df. isnull( ) . sum ( )
df[ [ 'Age' , 'Cabin' ] ]
处理缺失值
df[ df[ 'Age' ] == None ] = 0
df[ df[ 'Age' ] . isnull( ) ] = 0
df[ df[ 'Age' ] == np. nan] = 0
df. dropna( ) . head( )
df. fillna( 0 ) . head( )
重复值
df[ df. duplicated( ) ]
df= df. drop_duplicates( )
df. head( )
df. to_csv( 'data.csv' )
连续变量离散化
df[ 'age_cut' ] = pd. cut( df[ 'Age' ] , 5 , lables= [ 1 , 2 , 3 , 4 , 5 ] )
df. head( )
df[ 'age_cut' ] = pd. cut( df[ 'Age' ] , [ 0 , 5 , 15 , 30 , 50 , 80 ] , lables= [ 1 , 2 , 3 , 4 , 5 ] )
df[ 'AgeBand' ] = pd. qcut( df[ 'Age' ] , [ 0 , 0.1 , 0.3 , 0.5 , 0.7 , 0.9 ] , labels = [ 1 , 2 , 3 , 4 , 5 ] )
一些分析
df[ 'Sex' ] . value_counts( )
df[ 'Sex' ] . unique( )
df[ 'Sex' ] . nunique( )
df[ 'Sex_num' ] = df[ 'Sex' ] . replace( [ 'male' , 'female' ] , [ 1 , 2 ] )
df[ 'Title' ] = df. Name. str . extract( '([A-Za-z]+)\.' , expand= False )
数据重构
list_up = [ text_left_up, text_right_up]
result_up = pd. concat( list_up, axis= 1 )
result_up. head( )
resul_up = text_left_up. join( text_right_up)
result_down = text_left_down. join( text_right_down)
result = result_up. append( result_down)
result_up = pd. merge( text_left_up, text_right_up, left_index= True , right_index= True )
result_down = pd. merge( text_left_down, text_right_down, left_index= True , right_index= True )
result = resul_up. append( result_down)
unit_result = text. stack( ) . head( )
df = text[ 'Fare' ] . groupby( text[ 'Sex' ] )
df = text[ 'Fare' ] . groupby( text[ 'Sex' ] ) . sum ( )
df. mean( )
text. groupby( 'Sex' ) . agg( { 'Fare' : 'mean' , 'Pclass' : 'count' } ) . rename( columns=
{ 'Fare' : 'mean_fare' , 'Pclass' : 'count_pclass' } )
text. groupby( [ 'A' , 'B' ] ) [ 'Fare' ] . mean( ) . head( )
result = pd. merge( means, survived_sex, on= 'Sex' )
survived_age = text[ 'Survived' ] . groupby( text[ 'Age' ] ) . sum ( )
survived_age. head( )
survived_age[ survived_age. values== survived_age. max ( ) ]
_sum = text[ 'Survived' ] . sum ( )
print ( _sum)
_sum = text[ 'Survived' ] . sum ( )
print ( "sum of person:" + str ( _sum) )
precetn = survived_age. max ( ) / _sum
print ( "最大存活率:" + str ( precetn) )
数据可视化
import numpy as np
import pandas as pd
import matplotlib. pyplot as plt
sex = text. groupby( 'Sex' ) [ 'Survived' ] . sum ( )
sex. plot. bar( )
plt. title( 'survived_count' )
plt. show( )
text. groupby( [ 'Sex' , 'Survived' ] ) [ 'Survived' ] . count( ) . unstack( ) . plot( kind= 'bar' , stacked= 'True' )
plt. title( 'survived_count' )
plt. ylabel( 'count' )
fare_sur = text. groupby( [ 'Fare' ] ) [ 'Survived' ] . value_counts( ) . sort_values( ascending= False )
fare_sur
fig = plt. figure( figsize= ( 20 , 18 ) )
fare_sur. plot( grid= True )
plt. show( )
fare_sur1 = text. groupby( [ 'Fare' ] ) [ 'Survived' ] . value_counts( )
fare_sur1
fig = plt. figure( figsize= ( 20 , 18 ) )
fare_sur1. plot( grid= True )
plt. legend( )
plt. show( )
pclass_sur = text. groupby( [ 'Pclass' ] ) [ 'Survived' ] . value_counts( )
pclass_sur
import seaborn as sns
sns. countplot( x= "Pclass" , hue= "Survived" , data= text)
facet = sns. FacetGrid( text, hue= "Survived" , aspect= 3 )
facet. map ( sns. kdeplot, 'Age' , shade= True )
facet. set ( xlim= ( 0 , text[ 'Age' ] . max ( ) ) )
facet. add_legend( )
text. Age[ text. Pclass == 1 ] . plot( kind= 'kde' )
text. Age[ text. Pclass == 2 ] . plot( kind= 'kde' )
text. Age[ text. Pclass == 3 ] . plot( kind= 'kde' )
plt. xlabel( "age" )
plt. legend( ( 1 , 2 , 3 ) , loc= "best" )
建模
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib. pyplot as plt
from IPython. display import Image
plt. rcParams[ 'font.sans-serif' ] = [ 'SimHei' ]
plt. rcParams[ 'axes.unicode_minus' ] = False
plt. rcParams[ 'figure.figsize' ] = ( 10 , 6 )
train = pd. read_csv( 'train.csv' )
train. shape
train[ 'Cabin' ] = train[ 'Cabin' ] . fillna( 'NA' )
train[ 'Embarked' ] = train[ 'Embarked' ] . fillna( 'S' )
train[ 'Age' ] = train[ 'Age' ] . fillna( train[ 'Age' ] . mean( ) )
train. isnull( ) . sum ( ) . sort_values( ascending= False )
data = train[ [ 'Pclass' , 'Sex' , 'Age' , 'SibSp' , 'Parch' , 'Fare' , 'Embarked' ] ]
data = pd. get_dummies( data)
rom sklearn. model_selection import train_test_split
X = data
y = train[ 'Survived' ]
X_train, X_test, y_train, y_test = train_test_split( X, y, stratify= y, random_state= 0 )
X_train. shape, X_test. shape
from sklearn. linear_model import LogisticRegression
from sklearn. ensemble import RandomForestClassifier
lr = LogisticRegression( )
lr. fit( X_train, y_train)
print ( "Training set score: {:.2f}" . format ( lr. score( X_train, y_train) ) )
print ( "Testing set score: {:.2f}" . format ( lr. score( X_test, y_test) ) )
lr2 = LogisticRegression( C= 100 )
lr2. fit( X_train, y_train)
print ( "Training set score: {:.2f}" . format ( lr2. score( X_train, y_train) ) )
print ( "Testing set score: {:.2f}" . format ( lr2. score( X_test, y_test) ) )