基于用户搜索关键词数据为用户打上标签（年龄，性别，学历）
整体流程如下：
文章目录
1.数据预处理
1.1 编码方式转换
- 将原始数据转换成utf-8编码，防止后续出现各种编码问题¶
由于原始数据比较大，在分词与过滤阶段会比较慢，这里我们选择了原始数据中的1W个
import csv
#原始数据存储路径
data_path = './data/user_tag_query.10W.TRAIN'
#生成数据路径
csvfile = open(data_path + '-1w.csv', 'w')
writer = csv.writer(csvfile)
writer.writerow(['ID', 'age', 'Gender', 'Education', 'QueryList'])
#转换成utf-8编码的格式
with open(data_path, 'r',encoding='gb18030',errors='ignore') as f:
lines = f.readlines()
print(lines)
for line in lines[0:10000]:
try:
line.strip()
data = line.split("\t")
writedata = [data[0], data[1], data[2], data[3]]
querystr = ''
data[-1]=data[-1][:-1]
for d in data[4:]:
try:
cur_str = d.encode('utf8')
cur_str = cur_str.decode('utf8')
querystr += cur_str + '\t'
except:
continue
#print (data[0][0:10])
querystr = querystr[:-1]
writedata.append(querystr)
writer.writerow(writedata)
except:
#print (data[0][0:20])
continue
- 测试集的编码转换方式同上
data_path = './data/user_tag_query.10W.TEST'
csvfile = open(data_path + '-1w.csv', 'w')
writer = csv.writer(csvfile)
writer.writerow(['ID', 'QueryList'])
with open(data_path, 'r',encoding='gb18030',errors='ignore') as f:
lines = f.readlines()
for line in lines[0:10000]:
try:
data = line.split("\t")
writedata = [data[0]]
querystr = ''
data[-1]=data[-1][:-1]
for d in data[1:]:
try:
cur_str = d.encode('utf8')
cur_str = cur_str.decode('utf8')
querystr += cur_str + '\t'
except:
#print (data[0][0:10])
continue
querystr = querystr[:-1]
writedata.append(querystr)
writer.writerow(writedata)
except:
#print (data[0][0:20])
continue
- 生成对应的数据表
import pandas as pd
#编码转换完成的数据，取的是1W的子集
trainname = './data/user_tag_query.10W.TRAIN-1w.csv'
testname = './data/user_tag_query.10W.TEST-1w.csv'
data = pd.read_csv(trainname,encoding='gbk')
print (data.info())
print(data.head())
#分别生成三种标签数据（性别，年龄，学历）
data.age.to_csv("./data/train_age.csv", index=False)
data.Gender.to_csv("./data/train_gender.csv", index=False)
data.Education.to_csv("./data/train_education.csv", index=False)
#将搜索数据单独拿出来
data.QueryList.to_csv("./data/train_querylist.csv", index=False)
data = pd.read_csv(testname,encoding='gbk')
print (data.info())
data.QueryList.to_csv("./data/test_querylist.csv", index=False)
1.2 对数据搜索内容进行分词与词性过滤
- 这里需要分别对训练集和测试集进行相同的操作，路径名字要改动一下
import pandas as pd
import jieba.analyse
import time
import jieba
import jieba.posseg
import os, sys
def input(trainname):
traindata = []
with open(trainname, 'rb') as f:
line = f.readline()
count = 0
while line:
try:
traindata.append(line)
count += 1
except:
print ("error:", line, count)
line=f.readline()
return traindata
start = time.clock()
filepath = './data/test_querylist.csv'
QueryList = input(filepath)
writepath = './data/test_querylist_writefile-1w.csv'
csvfile = open(writepath, 'w')
POS = {}
for i in range(len(QueryList)):
#print (i)
if i%2000 == 0 and i >=1000:
print (i,'finished')
s = []
str = ""
words = jieba.posseg.cut(QueryList[i])# 带有词性的精确分词模式
allowPOS = ['n','v','j']
for word, flag in words:
POS[flag]=POS.get(flag,0)+1
if (flag[0] in allowPOS) and len(word)>=2:
str += word + " "
cur_str = str.encode('utf8')
cur_str = cur_str.decode('utf8')
s.append(cur_str)
csvfile.write(" ".join(s)+'\n')
csvfile.close()
end = time.clock()
print ("total time: %f s" % (end - start))
2.特征选择
2.1 建立word2vec词向量模型
参数定义：
-
sentences：可以是一个list
-
sg： 用于设置训练算法，默认为0，对应CBOW算法；sg=1则采用skip-gram算法。
-
size：是指特征向量的维度，默认为100。大的size需要更多的训练数据,但是效果会更好. 推荐值为几十到几百。
-
window：表示当前词与预测词在一个句子中的最大距离是多少
-
alpha: 是学习速率
-
seed：用于随机数发生器。与初始化词向量有关。
-
min_count: 可以对字典做截断. 词频少于min_count次数的单词会被丢弃掉, 默认值为5
-
max_vocab_size: 设置词向量构建期间的RAM限制。如果所有独立单词个数超过这个，则就消除掉其中最不频繁的一个。每一千万个单词需要大约1GB的RAM。设置成None则没有限制。
-
workers参数控制训练的并行数。
-
hs: 如果为1则会采用hierarchica·softmax技巧。如果设置为0（defau·t），则negative sampling会被使用。
-
negative: 如果>0,则会采用negativesamp·ing，用于设置多少个noise words
-
iter： 迭代次数，默认为5
from gensim.models import word2vec
#将数据变换成list of list格式
train_path = './data/train_querylist_writefile-1w.csv'
with open(train_path, 'r') as f:
My_list = []
lines = f.readlines()
for line in lines:
cur_list = []
line = line.strip()
data = line.split(" ")
for d in data:
cur_list.append(d)
My_list.append(cur_list)
model = word2vec.Word2Vec(My_list, size=300, window=10,workers=4)
savepath = '1w_word2vec_' + '300'+'.model' # 保存model的路径
model.save(savepath)
model.most_similar("大哥")
2.2 加载训练好的word2vec模型，对所有搜索数据求平均向量
import numpy as np
file_name = './data/train_querylist_writefile-1w.csv'
cur_model = gensim.models.Word2Vec.load('1w_word2vec_300.model')
with open(file_name, 'r') as f:
cur_index = 0
lines = f.readlines()
doc_cev = np.zeros((len(lines),300))
for line in lines:
word_vec = np.zeros((1,300))
words = line.strip().split(' ')
wrod_num = 0
#求模型的平均向量
for word in words:
if word in cur_model:
wrod_num += 1
word_vec += np.array([cur_model[word]])
doc_cev[cur_index] = word_vec / float(wrod_num)
cur_index += 1
doc_cev.shape
def removezero(x, y):
nozero = np.nonzero(y)
y = y[nozero]
x = np.array(x)
x = x[nozero]
return x, y
gender_train, genderlabel = removezero(doc_cev, genderlabel)
age_train, agelabel = removezero(doc_cev, agelabel)
education_train, educationlabel = removezero(doc_cev, educationlabel)
print (gender_train.shape,genderlabel.shape)
print (age_train.shape,agelabel.shape)
print (education_train.shape,educationlabel.shape)
2.3 绘图函数，以性别为例，绘制混淆矩阵
import matplotlib.pyplot as plt
import itertools
def plot_confusion_matrix(cm, classes,
title='Confusion matrix',
cmap=plt.cm.Blues):
"""
This function prints and plots the confusion matrix.
"""
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=0)
plt.yticks(tick_marks, classes)
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i, j],
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
2.4 测试集的构造方法和训练集一样
import numpy as np
file_name = './data/test_querylist_writefile-1w.csv'
cur_model = gensim.models.Word2Vec.load('1w_word2vec_300.model')
with open(file_name, 'r') as f:
cur_index = 0
lines = f.readlines()
doc_cev = np.zeros((len(lines),300))
for line in lines:
word_vec = np.zeros((1,300))
words = line.strip().split(' ')
wrod_num = 0
#求模型的平均向量
for word in words:
if word in cur_model:
wrod_num += 1
word_vec += np.array([cur_model[word]])
doc_cev[cur_index] = word_vec / float(wrod_num)
cur_index += 1
3.建模预测
3.1 建立基础模型LogisticRegression
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(gender_train,genderlabel,test_size = 0.2, random_state = 0)
#gender_train 为每个用户搜索的数据的词向量，genderlabel 为用户的性别
LR_model = LogisticRegression()
LR_model.fit(X_train,y_train)
y_pred = LR_model.predict(X_test)
print (LR_model.score(X_test,y_test))
cnf_matrix = confusion_matrix(y_test,y_pred)
print("Recall metric in the testing dataset: ", cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))
print("accuracy metric in the testing dataset: ", (cnf_matrix[1,1]+cnf_matrix[0,0])/(cnf_matrix[0,0]+cnf_matrix[1,1]+cnf_matrix[1,0]+cnf_matrix[0,1]))
# Plot non-normalized confusion matrix
class_names = [0,1]
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=class_names, title='Gender-Confusion matrix')
plt.show()
可以看到已经成功的将准确率达到了81.7%，但是召回率确实只有77.2%，效果一般。这里只是以性别为例子，后续还可已将用户年龄、教育程度、喜好等分类出来给用户贴上标签，逐渐完善用户画像，具体得看业务需求。
本次的例子由于是类别样本分布不均匀导致效果一般，严格来说，任何数据集上都有数据不平衡现象，我们可以通过一下几点去解决类别分布不均匀的问题：
- 采样：
采样方法是通过对训练集进行处理使其从不平衡的数据集变成平衡的数据集，在大部分情况下会对最终的结果带来提升。采样分为上采样（Oversampling）和下采样（Undersampling），上采样是把小众类复制多份，下采样是从大众类中剔除一些样本，或者说只从大众类中选取部分样本。 - 数据合成
数据合成方法是利用已有样本生成更多样本，这类方法在小数据场景下有很多成功案例，比如医学图像分析等。
3.2 堆叠模型
from sklearn.svm import SVC
from sklearn.naive_bayes import MultinomialNB
clf1 = RandomForestClassifier(n_estimators=100,min_samples_split=5,max_depth=10)
clf2 = SVC()
clf3 = LogisticRegression()
basemodes = [
['rf', clf1],
['svm', clf2],
['lr', clf3]
]
from sklearn.cross_validation import KFold, StratifiedKFold
models = basemodes
#X_train, X_test, y_train, y_test
folds = list(KFold(len(y_train), n_folds=5, random_state=0))
print (len(folds))
S_train = np.zeros((X_train.shape[0], len(models)))
S_test = np.zeros((X_test.shape[0], len(models)))
for i, bm in enumerate(models):
clf = bm[1]
#S_test_i = np.zeros((y_test.shape[0], len(folds)))
for j, (train_idx, test_idx) in enumerate(folds):
X_train_cv = X_train[train_idx]
y_train_cv = y_train[train_idx]
X_val = X_train[test_idx]
clf.fit(X_train_cv, y_train_cv)
y_val = clf.predict(X_val)[:]
S_train[test_idx, i] = y_val
S_test[:,i] = clf.predict(X_test)
final_clf = RandomForestClassifier(n_estimators=100)
final_clf.fit(S_train,y_train)
print (final_clf.score(S_test,y_test))
1978
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
