数据科学学习笔记

第一部分

1、从文本文件中读取

import pandas as pd
color_table = pd.io.parsers.read_table("D:\\data\Colors.txt")
print(color_table)

2、读取CSV定界的格式

import pandas as pd
titanic = pd.io.parsers.read_csv("D:\\data\Titanic.csv")
X = titanic[['age']]
print(X)
#获取数据的优化
X = titanic[['age']].values
print(X)

3、读取Excel和其他的微软办公文件

import pandas as pd
trig_values = pd.read_excel("D:\\data\Values.xlsx",'Sheet1',index_col=None,na_values=['NA'])
print(trig_values)
#更高效和方便
xls = pd.ExcelFile("D:\\data\Values.xlsx")
trig_values = xls.parse('Sheet1',index_col=None,na_values=['NA'])
print(trig_values)

4、以非结构化文件的形式来发送数据(存在问题)

from skimage.io import imread
from skimage.transform import resize
from matplotlib import pyplot as plt
import matplotlib.cm as cm
example_file = ("http://upload.wikimedia.org/wikipedia/commons/7/7d/Dog_face.png")
image = imread(example_file, as_grey=True) #有颜色的图像转换成灰阶
plt.imshow(image, cmap=cm.gray) #灰阶颜色映射
plt.show()
#图像的更多信息
print("data type:%s,shape:%s"%(type(image),image.shape))
#裁剪图像
image2 = image[5:70,0:70]
plt.imshow(image, cmap=cm.gray)
plt.show()
#缩放图像
image3 = resize(image2, (30,30), mode='nearest')
plt.imshow(image3, cmap=cm.gray)
print("data type:%s,shape:%s"%(type(image3),image3.shape))
#扁平化处理图像
image_row = image3.flatten()
print("data type:%s,shape:%s"%(type(image_row),image_row.shape))

5、访问来自Web的数据

from lxml import objectify
import pandas as pd
xml = objectify.parse(open("D:\\data\XMLData.xml"))
root = xml.getroot()
df = pd.DataFrame(columns=('Number','String','Boolean'))
for i in range(0,4):
    obj = root.getchildren()[i].getchildren()
    row = dict(zip(['Number','String','Boolean'],[obj[0].text,obj[1].text,obj[2].text]))
    row_s = pd.Series(row)
    row_s.name = i
    df = df.append(row_s)
print(df)

第二部分

1、验证你的数据

#1.1 了解你的数据中有什么
from lxml import objectify
import pandas as pd
xml = objectify.parse(open("D:\\data\XMLData2.xml"))
root = xml.getroot()
df = pd.DataFrame(columns=('Number','String','Boolean'))
for i in range(0,6):
    obj = root.getchildren()[i].getchildren()
    row = dict(zip(['Number','String','Boolean'],[obj[0].text,obj[1].text,obj[2].text]))
    row_s = pd.Series(row)
    row_s.name = i
    df = df.append(row_s)
search = pd.DataFrame.duplicated(df) #duplicated:创建一个包含了重复行列表的搜素对象
print(df)
print()
print(search[search==True])

#1.2 去重
from lxml import objectify
import pandas as pd
xml = objectify.parse(open("D:\\data\XMLData2.xml"))
root = xml.getroot()
df = pd.DataFrame(columns=('Number','String','Boolean'))
for i in range(0,6):
    obj = root.getchildren()[i].getchildren()
    row = dict(zip(['Number','String','Boolean'],[obj[0].text,obj[1].text,obj[2].text]))
    row_s = pd.Series(row)
    row_s.name = i
    df = df.append(row_s)
print(df.drop_duplicates()) #drop_duplicates:移除记录

#1.3 数据规划
import pandas as pd
df = pd.DataFrame({'A':[0,0,0,0,0,1,1],'B':[1,2,3,5,4,2,5],'C':[5,3,4,1,1,2,3]})
a_group_desc = df.groupby('A').describe()
print(a_group_desc) #mean:均值 std:方差
#优化
unstacked = a_group_desc.unstack()
print(unstacked)
#减少信息输出量(存在问题)
print(unstacked.loc[:,(slice(None),['count','mean']),])

2、处理分类变量

#2.1 创建分类变量
import pandas as pd
car_colors = pd.Series(['Blue','Red','Green'],dtype='category')
car_data = pd.Series(pd.Categorical(['Yellow','Green','Red','Blue','Purple'],
                                   categories=car_colors,ordered=False))
find_entries = pd.isnull(car_data)
print(car_colors)
print()
print(car_data)
print()
print(find_entries[find_entries == True])

#2.2 重命名层级
import pandas as pd
car_colors = pd.Series(['Blue','Red','Green'],dtype='category')
car_data = pd.Series(pd.Categorical(['Blue','Green','Red','Blue','Red'],
                                   categories=car_colors,ordered=False))
car_colors.cat.categories = ['Purple','Yellow','Mauve']
car_data.cat.categories = car_colors
print(car_data)

#2.3 组合层级(存在问题)
import pandas as pd
car_colors = pd.Series(['Blue','Red','Green'],dtype='category')
car_data = pd.Series(pd.Categorical(['Blue','Green','Red','Green','Red','Green'],
                                   categories=car_colors,ordered=False))
car_data.cat.categories = ['Blue_Red','Red','Green']
print(car_data.loc[car_data.isin(['Red'])])
car_data.loc[car_data.isin(['Red'])] = 'Blue_Red'
print()
print(car_data)

3、处理你数据中的日期

#3.1 格式化日期和时间值
import datetime as dt
now = dt.datetime.now()
print(str(now))
print(now.strftime('%a,%d %B %Y'))

#3.2 使用正确的时间转换
import datetime as dt
now = dt.datetime.now()
timevalue = now + dt.timedelta(hours=2)
#days(),seconds(),microseconds(),milliseconds(),minutes(),hours(),weeks()
print(now.strftime('%H:%M:%S'))
print(timevalue.strftime('%H:%M:%S'))
print(timevalue-now)

4、处理丢失值

#4.1 寻找丢失的数据
import pandas as pd
import numpy as np
s = pd.Series([1,2,3,np.NaN,5,6,None])
print(s.isnull())
print()
print(s[s.isnull()])

#4.2 为丢失项编码
import pandas as pd
import numpy as np
s = pd.Series([1,2,3,np.NaN,5,6,None])
print(s.fillna(int(s.mean()))) #填充丢失项
print()
print(s.dropna()) #丢弃丢失项

#4.3 为丢失数据估值(存在问题)
import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
s = pd.Series([1,2,3,np.NaN,5,6,None])
#mean:平均数 median:中位数 most_frequent:重数
imp = SimpleImputer(missing_values='NaN',strategy='mean',axis=0)
imp.fit([1,2,3,4,5,6,7])
x = pd.Series(imp.transform(s).tolist()[0])
print(x)

5、交叉分析：过滤并选取数据

    #5.1 切分行
import numpy as np
x = np.array([[[1,2,3],[4,5,6],[7,8,9]],
             [[11,12,13],[14,15,16],[17,18,19]],
              [[21,22,23],[24,25,26],[27,28,29]]])
print(x[1])

    #5.2 切分列
import numpy as np
x = np.array([[[1,2,3],[4,5,6],[7,8,9]],
             [[11,12,13],[14,15,16],[17,18,19]],
              [[21,22,23],[24,25,26],[27,28,29]]])
print(x[:,1])

    #5.3 切块
import numpy as np
x = np.array([[[1,2,3],[4,5,6],[7,8,9]],
             [[11,12,13],[14,15,16],[17,18,19]],
              [[21,22,23],[24,25,26],[27,28,29]]])
print(x[1,1])
print(x[:,1,1])
print(x[1,:,1])
print()
print(x[1:3,1:3])

6、连接和变换

#6.1 增加新的实例和变量
import pandas as pd
df = pd.DataFrame({'A':[2,3,1],
                  'B':[1,2,3],
                  'C':[5,3,4]})
df1 = pd.DataFrame({'A':[4],
                  'B':[4],
                  'C':[4]})
df = df.append(df1)
df = df.reset_index(drop=True)
print(df)
df.loc[df.last_valid_index()+1] = [5,5,5]
print()
print(df)
df2 = pd.DataFrame({'D':[1,2,3,4,5]})
df = pd.DataFrame.join(df,df2)
print()
print(df)

#6.2 移除数据
import pandas as pd
df = pd.DataFrame({'A':[2,3,1],
                  'B':[1,2,3],
                  'C':[5,3,4]})
df = df.drop(df.index[[1]])
print(df)
df = df.drop('B',1)
print()
print(df)

#6.3 排序和搅乱
import pandas as pd
import numpy as np
df = pd.DataFrame({'A':[2,1,2,3,3,5,4],
                  'B':[1,2,3,5,4,2,5],
                  'C':[5,3,4,1,1,2,3]})
df = df.sort_values(by=['A','B'],ascending=[True,True])
df = df.reset_index(drop=True)
print(df)
index = df.index.tolist()
np.random.shuffle(index)
df = df.iloc[index]
df = df.reset_index(drop=True)
print()
print(df)

7、在任何层次聚合数据

import pandas as pd
df = pd.DataFrame({'Map':[0,0,0,1,1,2,2],
                  'Values':[1,2,3,5,4,2,5]})
df['S'] = df.groupby('Map')['Values'].transform(np.sum)
df['M'] = df.groupby('Map')['Values'].transform(np.mean)
df['V'] = df.groupby('Map')['Values'].transform(np.var)
print(df)