机器学习特征工程基本流程

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import MinMaxScaler,StandardScaler
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA
from scipy.stats import pearsonr
import jieba
import pandas as pd

1.获取数据集

def datasets_demo():
    '''
    sklearn数据集使用
    ：return:
    '''
    #获取数据集
    iris=load_iris()
    print('鸢尾花数据集:\n',iris)
    print('查看数据集描述:\n',iris['DESCR'])
    print('查看特征值的名字:\n',iris.feature_names)
    print('查看特征值:\n',iris.data,iris.data.shape)
    #数据集划分
    x_train,x_test,y_train,y_test=train_test_split(iris.data,iris.target,
                                                   test_size=0.2,random_state=666)
    print('训练集的特征值:\n',x_train,x_train.shape)

    return None
datasets_demo()

2.特征提取
（1）英文

def dict_demo():
    '''
    字典特征抽取
    '''
    data = [{'city':'北京','temperature':100},{'city':'上海','temperature':60}, {'city':'深圳','temperature':30}]
    # 1、实例化一个转换器类
    transfer=DictVectorizer(sparse=True)
    #2.调用fit_transform()
    data_new=transfer.fit_transform(data)
    print('data_new:\n',data_new.toarray(),type(data_new))
    print('特征名字:\n',transfer.get_feature_names())
    return None
dict_demo()

def count_demo():
    """
    文本特征抽取：CountVecotrizer
    :return:
    """
    data = ["life is short,i like like python", "life is too long,i dislike python"]
    # 1、实例化一个转换器类
    transfer = CountVectorizer(stop_words=["is", "too"])

    # 2、调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new:\n", data_new.toarray())
    print("特征名字：\n", transfer.get_feature_names())

    return None
count_demo()

（2）中文

def cut_word(text):
    '''
    进行中文分词：‘我爱北京天安门’
    '''
    return ' '.join(list(jieba.cut(text)))
text='我爱北京天安门'
cut_word(text)

out:'我 爱 北京 天安门'

def count_chinese_demo2():
    # 将中文文本进行分词
    data = ["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。",
            "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去。",
            "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系。"]
    data_new=[]
    for sent in data:
        data_new.append(cut_word(sent))
    #1.实例化一个转换器
    transfer=CountVectorizer(stop_words=['一种','所以'])
    #调用fit_transform
    data_final=transfer.fit_transform(data_new)
    print("data_new:\n", data_final.toarray())
    print("特征名字：\n", transfer.get_feature_names())

    return None
count_chinese_demo2()

def tfidf_demo():
    """
    用TF-IDF的方法进行文本特征抽取
    :return:
    """
    # 将中文文本进行分词
    data = ["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。",
            "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去。",
            "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系。"]
    data_new=[]
    for sent in data:
        data_new.append(cut_word(sent))
    # 1、实例化一个转换器类
    transfer = TfidfVectorizer(stop_words=["一种", "所以"])
    # 2、调用fit_transform
    data_final = transfer.fit_transform(data_new)
    print("data_new:\n", data_final.toarray())
    print("特征名字：\n", transfer.get_feature_names())

    return None

tfidf_demo() 

3.归一化

def minmax_demo():
    #获取数据
    data=pd.read_csv('dating.txt')
    data=data.iloc[:,:3]
    print('data:\n',data)
    #实例化一个转换器
    transfer=MinMaxScaler(feature_range=[2,3])
    #调用fit_transform
    data_new=transfer.fit_transform(data)
    print('data_new:\n',data_new)
    return None
minmax_demo()

4.标准化

def stand_demo():
    #1.获取数据
    data=pd.read_csv('dating.txt')
    data=data.iloc[:,:3]
    #实例化一个转换器
    transfer=StandardScaler()
    data_new=transfer.fit_transform(data)
    print("data_new:\n", data_new)
    return None
stand_demo()  

5.过滤低方差特征
过滤前数据

data=pd.read_csv('factor_returns.csv')
data.head()

def variance_demo():
    #1.获取数据
    data=pd.read_csv('factor_returns.csv')
    data=data.iloc[:,1:-2]
    #2.实例化一个转换器类
    transfer=VarianceThreshold(threshold=10)
    #3.调用fit_transform
    data_new=transfer.fit_transform(data)
    print('data_new:\n',data_new)
    print(data_new.shape)
variance_demo()

6.过滤相关性数据

#计算某两个变量之间的相关系数
def pearsonr_demo():
    data = pd.read_csv('factor_returns.csv')
    data = data.iloc[:, 1:-2]
    r1=pearsonr(data["pe_ratio"], data["pb_ratio"])
    print('相关系数:\n',r1)
    r2=pearsonr(data['revenue'],data['total_expense'])
    print("相关系数：\n", r2)
    return None
pearsonr_demo()

#循环计算两个变量之间的相关系数
def pearsonr_demo2():
     data = pd.read_csv('factor_returns.csv')
     data = data.iloc[:,1:-2]
     factor=['pe_ratio','pb_ratio','market_cap','return_on_asset_net_profit','du_return_on_equity','ev','earnings_per_share','revenue','total_expense']
     for i in range(len(factor)):
            for j in range(i,len(factor)-1):
                r=pearsonr(data[(factor[i])],data[(factor[j+1])])
#                 print(r)  注意这儿的相关系数得出来的结果是一个tuple元组
                print('指标%s和指标%s之间的相关系数为%f'%(factor[i],factor[j+1],r[0]))
     return None
pearsonr_demo2()

7.pca降维

def pca_demo():
    data=[[2,8,4,5], [6,3,0,8], [5,4,9,1]]
    #实例化一个转化器
    transfer=PCA(n_components=0.95)
    #调用fit_tranform
    data_new=transfer.fit_transform(data)
    print('data_new:\n',data_new)
    return None
pca_demo()