前言
缺失值or异常值的处理是特征工程中最常见的问题。缺失值的存在使得常用的分析与算法受阻。异常值的存在会扰乱统计分析与机器学习的结果,使得模型更偏向于对异常值的过拟合,失去泛化能力。
所以在数据预处理阶段,我们要根据实际业务来判断如何处理缺失值与异常值。本文整理了对两者的判断实现方法和处理方法,希望大家能结合实际情况灵活运用。
数据准备
我们以一个电信数据集为例
百度网盘下载:
链接:https://pan.baidu.com/s/1YT3mKxQpAXEkP-e6ZcW5Vg
提取码:yavw
pacman::p_load(caret, tidyverse, data.table)
data <- fread("D://contest//dianxin//dianxin.csv", stringsAsFactors = T) %>% tbl_df()
data_na <- data
#看一下数据维度:18513条记录 、 14个变量
> names(data)
[1] "id" "state" "account_length" "area_code"
[5] "phone_number" "international_plan" "voice_mail_plan" "number_vmail_messages"
[9] "total_day_minutes" "total_day_calls" "total_day_charge" "total_eve_minutes"
[13] "total_eve_calls" "total_eve_charge" "total_night_minutes" "total_night_calls"
[17] "total_night_charge" "total_intl_minutes" "total_intl_calls" "total_intl_charge"
[21] "Churn"
> dim(data)
[1] 600 21
#将缺失值随机插入到每列(除了"id"和"Churn")
set.seed(1)
for (i in 2:(ncol(data) - 1)) {
data_na[sample(1:600, size = sample(1:50, size = 1)), i] <- NA
}
缺失值
1
可视化缺失情况
DataExplorer::plot_missing(data_na)
VIM::aggr(data_na)
2
利用某一值插补缺失
#变量自身均值插补
data_na$account_length <- tidyr::replace_na(data_na$account_length, mean(data_na$account_length, na.rm = T))
#account_length列插补前
> summary(data_na$account_length)
Min. 1st Qu. Median Mean 3rd Qu. Max. NA's
1.0 73.0 100.0 100.6 127.0 232.0 14
#插补后
> summary(data_na$account_length)
Min. 1st Qu. Median Mean 3rd Qu. Max.
1.0 74.0 100.6 100.6 127.0 232.0
3
均值与knn插补 - caret
#均值插补,只对数值列生效
pre_na_mean <- caret::preProcess(data_na[, 2:19], method = "medianImpute", verbose = T) #均值法
newdata_mean <- predict(pre_na_mean, data_na[, 2:19])
> newdata_mean
# A tibble: 600 x 18
account_length area_code international_p~ voice_mail_plan number_vmail_me~ total_day_minut~ total_day_calls
&l