Python数据分析与机器学习-Pandas

源码下载地址：

http://download.csdn.net/download/adam_zs/10174600

import pandas

food_info = pandas.read_csv("food_info.csv")  # 默认第一行作为列名
print(food_info)
print(type(food_info))  # DataFrame
print(food_info.dtypes)

first_rows = food_info.head()  # 默认显示前5行
print(first_rows)
print(food_info.head(3))  # 显示前3行
print(food_info.tail(3))  # 显示后3行
print(food_info.shape)
print(food_info.columns)  # 列名
'''
['NDB_No', 'Shrt_Desc', 'Water_(g)', 'Energ_Kcal', 'Protein_(g)',
       'Lipid_Tot_(g)', 'Ash_(g)', 'Carbohydrt_(g)', 'Fiber_TD_(g)',
       'Sugar_Tot_(g)', 'Calcium_(mg)', 'Iron_(mg)', 'Magnesium_(mg)',
       'Phosphorus_(mg)', 'Potassium_(mg)', 'Sodium_(mg)', 'Zinc_(mg)',
       'Copper_(mg)', 'Manganese_(mg)', 'Selenium_(mcg)', 'Vit_C_(mg)',
       'Thiamin_(mg)', 'Riboflavin_(mg)', 'Niacin_(mg)', 'Vit_B6_(mg)',
       'Vit_B12_(mcg)', 'Vit_A_IU', 'Vit_A_RAE', 'Vit_E_(mg)', 'Vit_D_mcg',
       'Vit_D_IU', 'Vit_K_(mcg)', 'FA_Sat_(g)', 'FA_Mono_(g)', 'FA_Poly_(g)',
       'Cholestrl_(mg)']
'''
print(food_info.loc[0])  # 第一行
print(food_info.loc[6])  # 第七行
# print(food_info.loc[8620]) #KeyError: 'the label [8620] is not in the [index]'

print(food_info.loc[3:6])  # 切片，返回3,4,5,6行，包括第3行和第6行
print(food_info.loc[[2, 5, 10]])  # 返回2, 5, 10行
print(food_info["NDB_No"])  # 显示NDB_No列数据
print(food_info[["Zinc_(mg)", "Copper_(mg)"]])  # 显示Zinc_(mg),Copper_(mg)列数据

col_names = food_info.columns.tolist()
gram_columns = []
for c in col_names:
    if c.endswith("(g)"):
        gram_columns.append(c)
gram_df = food_info[gram_columns]
print(gram_df.head(3))

import pandas

food_info = pandas.read_csv("food_info.csv")
col_names = food_info.columns.tolist()
print(col_names)
# ['NDB_No', 'Shrt_Desc', 'Water_(g)', 'Energ_Kcal', 'Protein_(g)', 'Lipid_Tot_(g)', 'Ash_(g)',
# #'Carbohydrt_(g)', 'Fiber_TD_(g)', 'Sugar_Tot_(g)', 'Calcium_(mg)', 'Iron_(mg)', 'Magnesium_(mg)',
# 'Phosphorus_(mg)', 'Potassium_(mg)', 'Sodium_(mg)', 'Zinc_(mg)', 'Copper_(mg)', 'Manganese_(mg)',
# 'Selenium_(mcg)', 'Vit_C_(mg)', 'Thiamin_(mg)', 'Riboflavin_(mg)', 'Niacin_(mg)', 'Vit_B6_(mg)',
# 'Vit_B12_(mcg)', 'Vit_A_IU', 'Vit_A_RAE', 'Vit_E_(mg)', 'Vit_D_mcg', 'Vit_D_IU', 'Vit_K_(mcg)',
# 'FA_Sat_(g)', 'FA_Mono_(g)', 'FA_Poly_(g)', 'Cholestrl_(mg)']
print(food_info[col_names])

print(food_info["Iron_(mg)"])
print(food_info["Iron_(mg)"] / 1000)
print(food_info["Iron_(mg)"] + 100)
print(food_info["Iron_(mg)"] - 100)
print(food_info["Iron_(mg)"] * 2)

# 对应位置相乘、相除
water_energy = food_info["Water_(g)"] * food_info["Energ_Kcal"]
iron_grams = food_info["Iron_(mg)"] / 1000
food_info["Iron_(g)"] = iron_grams

weighted_protein = food_info["Protein_(g)"] * 2
weighted_fat = -0.75 * food_info["Lipid_Tot_(g)"]
initial_rating = weighted_protein + weighted_fat

print(food_info["Energ_Kcal"])
max_calories = food_info["Energ_Kcal"].max()
normalized_calories = food_info["Energ_Kcal"] / max_calories
normalized_protein = food_info["Protein_(g)"] / food_info["Protein_(g)"].max()
normalized_fat = food_info["Lipid_Tot_(g)"] / food_info["Lipid_Tot_(g)"].max()
food_info["Normalized_Protein"] = normalized_protein
food_info["Normalized_Fat"] = normalized_fat

# inplace=True,对DataFrame进行就地排序，而不是返回新的DataFrame。
food_info.sort_values("Sodium_(mg)", inplace=True, ascending=True)  # 升序排列,默认升序
print(food_info["Sodium_(mg)"])
food_info.sort_values("Sodium_(mg)", inplace=True, ascending=False)  # 降序排列
print(food_info["Sodium_(mg)"])

import pandas as pd
import numpy as np

# 泰克尼克号获救信号
'''
'PassengerId' 乘客id
 'Survived' 标签值
 'Pclass' 仓位等级
 'Name' 姓名
 'Sex' 性别
 'Age' 年龄
 'SibSp' 兄弟姐妹数量
 'Parch' 老人孩子总数
 'Ticket' 票号
 'Fare' 票价
 'Cabin' 座位
 'Embarked' 登船地点
'''

# pd.set_option('display.height', 1000)
# pd.set_option('display.max_rows', 500)
# pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

titanic_survival = pd.read_csv("titanic_train.csv")
print(titanic_survival.columns)
print(titanic_survival.head())

# The Pandas library uses NaN, which stands for "not a number", to indicate a missing value.
# we can use the pandas.isnull() function which takes a pandas series and returns a series of True and False values
# Age没有值的数量
age = titanic_survival["Age"]
print(age)
age_is_null = pd.isnull(age)  # pd.isnull true:是缺失值 flase:不是缺失值
print(age_is_null)
age_null_true = age[age_is_null]
print(len(age_null_true))  # Age缺失的数量 177

mean_age = sum(titanic_survival["Age"]) / len(titanic_survival["Age"])
print(mean_age)  # nan

good_ages = titanic_survival["Age"][age_is_null == False]
print(sum(good_ages) / len(good_ages))  # 29.6991176471

correct_mean_age = titanic_survival["Age"].mean()  # mean,平均年龄,自动筛选出没有age的数据
print(correct_mean_age)  # 29.69911764705882

# 每个仓位票价的均价
passenger_classes = [1, 2, 3]
fares_by_class = {}
for this_class in passenger_classes:
    pclass_rows = titanic_survival[titanic_survival["Pclass"] == this_class]
    fare_for_class = pclass_rows["Fare"].mean()
    fares_by_class[this_class] = fare_for_class
print(fares_by_class)

# pivot_table 数据统计
# index 按照那个分组
# values 是我们要应用计算的列
# aggfunc 指定我们要执行的计算
passenger_Fare = titanic_survival.pivot_table(index="Pclass", values="Fare", aggfunc=np.mean)
print(passenger_Fare)

passenger_Fare = titanic_survival.pivot_table(index="Pclass", values="Fare")  # 默认求平均值
print(passenger_Fare)

# 按照Embarked进行分组，求Fare,Survived的和
port_stats = titanic_survival.pivot_table(index="Embarked", values=["Fare", "Survived"], aggfunc=np.sum)
print(port_stats)

print(titanic_survival)
# 指定axis= 1或axis='columns' 删除任何具有空值的列
print(titanic_survival.fillna(0))  # 缺失值都填充为0
print(titanic_survival.dropna(axis=1))  # 删除有空值的列
print(titanic_survival.dropna(axis=0))  # 删除有空值的行
print(titanic_survival.dropna(axis=0, subset=["Age", "Sex"]))  # 删除"Age", "Sex"有空值的行

row_index_83_age = titanic_survival.loc[83, "Age"]  # 行号、列名
row_index_1000_pclass = titanic_survival.loc[766, "Pclass"]
print(titanic_survival.loc[83])
print(row_index_83_age)
print(titanic_survival.loc[766])
print(row_index_1000_pclass)

new_titanic_survival = titanic_survival.sort_values("Age", ascending=True)
print(new_titanic_survival)
print(new_titanic_survival.loc[:10])
itanic_reindexed = new_titanic_survival.reset_index(drop=True)  # 重建索引,返回重建索引后的矩阵
print(itanic_reindexed.loc[:10])


# 返回一系列的第一百个项目
def hundredth_row(column):
    return column.loc[99]


# apply运行自定义函数
hundredth_row = titanic_survival.apply(hundredth_row)
print(hundredth_row)


# 缺失值的数量
def not_null_count(column):
    column_null = pd.isnull(column)
    return len(titanic_survival[column_null])


print(pd.isnull(titanic_survival))
print(len(pd.isnull(titanic_survival)))  # 891
column_null_count = titanic_survival.apply(not_null_count, axis=0)  # axis=0 列,axis=1 行
print(column_null_count)


def which_class(row):
    pclass = row["Pclass"]
    if pd.isnull(pclass):
        return "Unknown"
    elif pclass == 1:
        return "First Class"
    elif pclass == 2:
        return "Second Class"
    elif pclass == 3:
        return "Third Class"


classes = titanic_survival.apply(which_class, axis=1)
print(classes)


def is_minor(row):
    if row["Age"] > 18:
        return True
    else:
        return False


ages = titanic_survival.apply(is_minor, axis=1)
print(ages)


def generate_age_label(row):
    age = row["Age"]
    if pd.isnull(age):
        return "unknown"
    elif age < 18:
        return "minor"
    else:
        return "adult"


age_labels = titanic_survival.apply(generate_age_label, axis=1)
print(age_labels)

titanic_survival['age_labels'] = age_labels
# 成年、未成年的获救率
age_group_survival = titanic_survival.pivot_table(index="age_labels", values="Survived")
print(age_group_survival)

import pandas as pd

pd.set_option('display.height', 9999)
pd.set_option('display.max_rows', 9999)
pd.set_option('display.max_columns', 9999)
pd.set_option('display.width', 9999)

# Series (collection of values) 一行或者一列
# DataFrame (collection of Series objects)
# Panel (collection of DataFrame objects)

# DataFrame中一行或者一列是 Series
# Series.values 是 ndarray

# 电影评分
fandango = pd.read_csv('fandango_score_comparison.csv')
print(fandango.head())
series_film = fandango['FILM']
print(type(series_film))  # Series
print(series_film[0:5])
series_rt = fandango['RottenTomatoes']
print(series_rt[0:5])

from pandas import Series

film_names = series_film.values
print(type(film_names))  # ndarray
print(film_names)
rt_scores = series_rt.values
print(rt_scores)
# 电影名字film_names作为索引
series_custom = Series(rt_scores, index=film_names)
print(series_custom)
print(series_custom[['Minions (2015)', 'Leviathan (2014)']])

# int index is also aviable
series_custom = Series(rt_scores, index=film_names)
print(series_custom)
print(series_custom[['Minions (2015)', 'Leviathan (2014)']])
fiveten = series_custom[5:10]
print(fiveten)

original_index = series_custom.index.tolist()
print(original_index)
sorted_index = sorted(original_index)
print(sorted_index)
sorted_by_index = series_custom.reindex(sorted_index)
print(sorted_by_index)

sc2 = series_custom.sort_index()
sc3 = series_custom.sort_values()
print(sc2[:10])
print(sc3[:10])

import numpy as np

print(series_custom.head())
# Add each value with each other
print(np.add(series_custom.head(), series_custom.head()))
# Apply sine function to each value
print(np.sin(series_custom))
# Return the highest value (will return a single value not a Series)
print(np.max(series_custom))

print(fandango.head())
series_custom = Series(fandango['RottenTomatoes'].values, index=fandango['FILM'].values)
print(series_custom.head())
print(series_custom > 50)

series_greater_than_50 = series_custom[series_custom > 50]
criteria_one = series_custom > 50
criteria_two = series_custom < 75
both_criteria = series_custom[criteria_one & criteria_two]
print(both_criteria)

# data alignment same index
rt_critics = Series(fandango['RottenTomatoes'].values, index=fandango['FILM'])
rt_users = Series(fandango['RottenTomatoes_User'].values, index=fandango['FILM'])
rt_mean = (rt_critics + rt_users) / 2
print(rt_mean)

import pandas as pd

pd.set_option('display.height', 9999)
pd.set_option('display.max_rows', 9999)
pd.set_option('display.max_columns', 9999)
pd.set_option('display.width', 9999)

fandango = pd.read_csv('fandango_score_comparison.csv')
print(fandango.head())
print(type(fandango))

# DataFrame指定索引
fandango_films = fandango.set_index('FILM', drop=True)  # 以FILM作为索引，删除原FILM的列
print(fandango_films)
print(fandango_films.index)

# Slice using either bracket notation or loc[]
print(fandango_films.loc["Avengers: Age of Ultron (2015)":"Hot Tub Time Machine 2 (2015)"])
print(fandango_films.loc["Avengers: Age of Ultron (2015)":"Hot Tub Time Machine 2 (2015)"])

# Specific movie
print(fandango_films.loc['Kumiko, The Treasure Hunter (2015)'])

# Selecting list of movies
movies = ['Kumiko, The Treasure Hunter (2015)', 'Do You Believe? (2015)', 'Ant-Man (2015)']
print(fandango_films.loc[movies])

# When selecting multiple rows, a DataFrame is returned,
# but when selecting an individual row, a Series object is returned instead

import numpy as np

print(fandango_films.head())
types = fandango_films.dtypes
print(types)
float_columns = types[types.values == 'float64'].index
float_df = fandango_films[float_columns]
print(float_df)

# `x` is a Series object representing a column
deviations = float_df.apply(lambda x: np.std(x))
print(deviations)

# np.std标准差
print(float_df.head())
rt_mt_user = float_df[['RT_user_norm', 'Metacritic_user_nom']]
print(rt_mt_user.apply(lambda x: np.std(x), axis=1))