【天池大赛-tf.keras】入门NLP新闻文本分类--采用（Word2vec+ GrandientBoosting）和（Word2vec+LSTM）两种方法！

【天池大赛–tf.keras】入门NLP - 新闻文本分类

赛题背景介绍和链接：零基础入门NLP - 新闻文本分类

方法1：Word2vec+ GrandientBoosting–83.93%

import pandas as pd
from os import path
import os
import re
import numpy as np
import logging
import pandas as pd

data_file = './data/train_set.csv'

f = pd.read_csv(data_file, sep='\t', encoding='UTF-8')
texts = f['text'].tolist()
labels = f['label'].tolist()
df = pd.DataFrame({'label': labels, 'txt': texts})
df.head()

num_features = 100
min_word_count = 10
num_workers = 4
context = 5
epoch = 20
sample = 1e-5

# word2vec
txt = df.txt.values
txtlist = []
for sent in txt:
    temp = [w for w in sent.split()]
    txtlist.append(temp)
    
print(txt[0])
print(txtlist[0])

from gensim.models import word2vec
#1.5min
model = word2vec.Word2Vec(txtlist, 
                          workers = num_workers,
                          sample = sample,
                          size = num_features,
                          min_count=min_word_count,
                          window = context,
                          iter = epoch)

# 用 word vec 的均值作为 doc vec
def get_doc_vec(sentence, model):
    scores = [model[word] for word in sentence.split() if word in model]  
    # 如果词频小于 min_count, word2vec 不会把这个词放入 vocab 里
    
    return np.mean(scores, axis=0)

from sklearn.model_selection import train_test_split
train_X, test_X, train_y, test_y = train_test_split(df.txt.values, df.label.values , train_size=0.9, random_state=1)

# 200sec
X_word2vec_train = np.array([get_doc_vec(sentence, model) for sentence in train_X])
X_word2vec_test = np.array([get_doc_vec(sentence, model) for sentence in test_X])

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
clf = GradientBoostingClassifier()

# 200sec
clf.fit(X_word2vec_train, train_y)
from sklearn import metrics
pre = clf.predict(X_word2vec_test)
print(metrics.classification_report(test_y, pre))

# 测试以及结果保存部分
import pandas as pd

data_file = './data/test_a.csv'

f = pd.read_csv(data_file, sep='\t', encoding='UTF-8')
texts = f['text'].tolist()
df = pd.DataFrame({'txt': texts})
df.head()
X_word2vec_finaltest1 = df.txt.values 

X_word2vec_finaltest = np.array([get_doc_vec(sentence, model) for sentence in X_word2vec_finaltest1])
final_pre = clf.predict(X_word2vec_finaltest)
print(final_pre)

save_test = './save_test.csv'
df = pd.DataFrame({'label':final_pre})
df.to_csv(save_test, index=False, sep=',')

方法2：Word2vec+LSTM

# 先导入需要的模块
import yaml
import keras
import sys
from sklearn.model_selection import train_test_split
import multiprocessing
import numpy as np
import pandas as pd
from keras.utils import to_categorical


from gensim.models.word2vec import Word2Vec
from gensim.corpora.dictionary import Dictionary

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.layers.core import Dense, Dropout,Activation
from keras.models import model_from_yaml

#np.random.seed(2019)

# 设定各类参数

vocab_dim = 200
maxlen = 1000
n_iterations = 20   #word2vec迭代次数
n_exposures = 10   #对字典做切断，词频小于min_count次数的单词会被丢弃
window_size = 7    #表示当前词与预测词在一个句子中的最大距离是多少


batch_size = 512          #LSTM的batch大小
n_epoch = 20             #遍历整个样本的次数
input_length = 1000      #LSTM的输入长度
cpu_count = multiprocessing.cpu_count()

import pandas as pd

# loaddata
def loaddata():
    data_file = './data/train_set.csv'
    f = pd.read_csv(data_file, sep='\t', encoding='UTF-8')
    texts = f['text'].tolist()
    labels = f['label'].tolist()
    df = pd.DataFrame({'label': labels, 'txt': texts})
    # word2vec
    combined = df.txt.values
    txtlist = []
    for sent in combined:
        temp = [w for w in sent.split()]
        txtlist.append(temp)
    y = df.label.values
    categories =['科技', '股票', '体育', '娱乐', '时政', '社会', '教育','财经', '家居', '游戏', '房产', '时尚', '彩票', '星座']
    cat_to_id = dict(zip(categories, range(len(categories))))
    return combined, txtlist, y, cat_to_id

# 创建字典
def create_dictionaries(model=None,combined=None):
    if (combined is not None) and (model is not None):
        gensim_dict = Dictionary()
        gensim_dict.doc2bow(model.wv.vocab.keys(),allow_update=True)
        w2indx = {v: k+1 for k, v in gensim_dict.items()}
        w2vec = {word: model[word] for word in w2indx.keys()}

        def parse_dataset(combined):
            data=[]
            for sentence in combined:
                new_txt = []
                for word in sentence:
                    try:
                        new_txt.append(w2indx[word])
                    except:
                        new_txt.append(0)
                data.append(new_txt)
            return data
        combined=parse_dataset(combined)
        combined= sequence.pad_sequences(combined, maxlen=maxlen)
        return w2indx, w2vec,combined
    else:
        print('无数据显示')

# word2vec训练
def word2vec_train(combined, txtlist):
    model = Word2Vec(size=vocab_dim,
                     min_count=n_exposures,
                     window=window_size,
                     workers=cpu_count,
                     iter=n_iterations)
    model.build_vocab(txtlist)
    model.train(txtlist, total_examples=model.corpus_count, epochs=model.iter)
    model.save('./w2v_example/LSTM_method/Word2vec_model.pkl')
    index_dict, word_vectors, combined = create_dictionaries(model=model,combined=combined)
    return index_dict, word_vectors, combined

# 获取数据：embedding_weights,x_train,y_train,x_test,y_test
def get_data(index_dict, word_vectors, combined, y, cat_to_id):
    n_symbols = len(index_dict) + 1
    print(n_symbols)
    embedding_weights = np.zeros((n_symbols, vocab_dim))
    for word, index in index_dict.items():
        embedding_weights[index, :] = word_vectors[word]
    x_train, x_test, y_train, y_test = train_test_split(combined, y, test_size=0.1)
    y_train = to_categorical(y_train,num_classes=len(cat_to_id))
    y_test = to_categorical(y_test, num_classes=len(cat_to_id))
    print(x_train.shape,y_train.shape)
    return n_symbols, embedding_weights, x_train, y_train, x_test, y_test

## 定义LSTM网络结构
def train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test):
    #定义贯序keras模型
    model = Sequential()
    model.add(Embedding(output_dim=vocab_dim,
                        input_dim=n_symbols,
                        mask_zero=True,
                        weights=[embedding_weights],
                        input_length=input_length))
    #model.add(LSTM(output_dim=50, activation='sigmoid', inner_activation='hard_sigmoid'))
    #model.add(Dropout(0.5))
    #model.add(Dense(14))
    #model.add(Activation('sigmoid'))
    model.add(LSTM(16, return_sequences=True))     
    model.add(LSTM(32, return_sequences=True))  
    model.add(LSTM(32, return_sequences=True))  
    model.add(LSTM(64, return_sequences=True))  
    model.add(LSTM(64)) 
    model.add(Dropout(0.3))
    model.add(Dense(14, activation='softmax'))

    #编译模型
    model.compile(loss=keras.losses.categorical_crossentropy,
                  optimizer=keras.optimizers.Adam(lr=0.001),
                  metrics=['accuracy'])

    print("Training...")
    model.fit(x_train, y_train, 
              batch_size=batch_size, 
              epochs=n_epoch,verbose=1,
              callbacks=[reduce_lr],
              validation_data=(x_test, y_test))

    #评估模型
    score = model.evaluate(x_test, y_test, batch_size=batch_size)

    yaml_string = model.to_yaml()
    with open('./w2v_example/LSTM_method/lstm.yml', 'w') as outfile:
        outfile.write( yaml.dump(yaml_string, default_flow_style=True) )
    model.save_weights('./w2v_example/LSTM_method/lstm.h5')
    print('Test score:', score)


# 训练模型，保存模型
def train():
    print("load data....")
    combined, txtlist, y, cat_to_id = loaddata()
    print('Word2vec模型训练中...')
    index_dict, word_vectors,combined=word2vec_train(combined, txtlist)
    print('设定Keras嵌入层中...')
    n_symbols,embedding_weights,x_train,y_train,x_test,y_test=get_data(index_dict, word_vectors, combined, y, cat_to_id)
    print(x_train.shape,y_train.shape)
    train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test)

from keras.callbacks import ReduceLROnPlateau

reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=5, mode='auto')

#train()
combined, txtlist, y, cat_to_id = loaddata()
model=Word2Vec.load('./w2v_example/LSTM_method/Word2vec_model.pkl')
index_dict, word_vectors,combined=create_dictionaries(model,combined=combined)
n_symbols,embedding_weights,x_train,y_train,x_test,y_test=get_data(index_dict, word_vectors, combined, y, cat_to_id)
print(x_train.shape,y_train.shape)
train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test)

方法二的predict part

import pandas as pd

data_file = './data/test_a.csv'

f = pd.read_csv(data_file, sep='\t', encoding='UTF-8')
texts = f['text'].tolist()
df = pd.DataFrame({'txt': texts})
df.head()
X_word2vec_finaltest1 = df.txt.values 

def input_transform(X_word2vec_finaltest1):
    model=Word2Vec.load('./w2v_example/LSTM_method/Word2vec_model.pkl')
    _,_,combined=create_dictionaries(model,X_word2vec_finaltest1)
    print(combined.shape)
    return combined

def lstm_predict():
    print('加载模型中...')
    with open('./w2v_example/LSTM_method/lstm.yml', 'r') as f:
        yaml_string = yaml.load(f)
    model = model_from_yaml(yaml_string)

    print('加载权重中...')
    # 权重加载，可以实时来做预测
    model.load_weights('./w2v_example/LSTM_method/lstm.h5')
    
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    return model

model = lstm_predict()
data=input_transform(X_word2vec_finaltest1)
# print(data)
result=model.predict_classes(data)
# print(result)

save_test = './save_test_lstm.csv'
df = pd.DataFrame({'label':final_result})
df.to_csv(save_test, index=False, sep=',')