一、数据插入
(1)数据补充
import numpy as np
import pandas as pd
from scipy.interpolate import interp1d
from scipy.interpolate import lagrange
x=[1,2,3,4,5,7,8]
y=[5,6,8,9,19,20,22]
lvl=lagrange(x,y)
lvl(6)
lagrange(x,y)(6)
lagrange(x,y)([6,11])
p=interp1d([1,2,3],[1,2,3],kind="linear")
p(1.2)
(2)实例
data=pd.read_csv(r"G:\大数据实验数据库\4.Python数据分析与应用\第5章\实训数据\missing_data.csv",header=None)
t0=data.iloc[:,1]
t1=t0[t0.notnull()]
t2=t1[len(t1)-5:]
lagrange(list(t2.index),list(t2))(20)
二、异常数据的识别与处理
(1)观察箱型图
import matplotlib.pyplot as plt
plt.figure(figsize=(10,8))
p=plt.boxplot(detail["counts"].values,notch=True)
o1=p["fliers"][0].get_ydata()
print(o1)
print(len(o1))
print(max(o1))
print(min(o1))
(2)3Q识别异常
detail=pd.read_csv(r"G:\大数据实验数据库\3.大数据实验数据\detail.csv",encoding="GBK")
ser1=detail["amounts"]
print("删除前的行数",ser1.shape)
k1=(ser1.mean()+3*ser1.std()>ser1)
k2=(ser1>ser1.mean()-3*ser1.std())
k3=k1&k2
for i in k3.index:
if k3[i]==False:
ser1.drop(i,inplace=True)
print("删除后的行数",ser1.shape)
三、数据标准化
(1)离差标准化
def minmaxscale(data):
data=(data-data.min())/(data.max()-data.min())
return data
d1=minmaxscale(detail["counts"])
d2=minmaxscale(detail["amounts"])
pd.concat([d1,d2],axis=1)
(2)标准差标准化
def stdscaler(data):
data=(data-data.min())/(data.std())
return data
d1=stdscaler(detail["counts"])
d2=stdscaler(detail["amounts"])
pd.concat([d1,d2],axis=1)
(3)小数定标标准化
def DecimalScaler(data):
data=data/10**np.ceil(np.log10(data.abs()/data.max()))
return data
d1=DecimalScaler(detail["counts"])
d2=DecimalScaler(detail["amounts"])
pd.concat([d1,d2],axis=1)
四、哑变量处理(文字)
import numpy as np
import pandas as pd
t1=pd.DataFrame(['广州',"上海","北京","武汉"],index=[1,2,3,4])
pd.get_dummies(t1[0])
t1=pd.DataFrame(['广州',"上海","北京","武汉"],index=[1,2,3,4],columns=["城市"])
pd.get_dummies(t1["城市"])
data=detail.loc[0:5,"dishes_name"]
pd.get_dummies(data)
五、数据离散化
(1)等宽法离散化 (max-min)/5
detail["amounts"].max()
detail["amounts"].min()
pd.cut(detail["amounts"],5)
(2)等频法离散化 number/5
data=detail["amounts"]
w=data.quantile(np.linspace(0,1,6))
pd.cut(data,w)