tensorflow + fasttext 实现文本分类及原理

1 fasttext原理简介

fasttext是2016年facebook的Joulin、Mikolov等人提出来的一种高效文本分类的算法，相比于其他神经网络模型而言，fasttext模型网络结构更加简单、运行起来更加快速，并且模型保持了较高的准确度。用论文原文的话说，训练10亿级别的数据量只需要10分钟。原文链接：https://arxiv.org/abs/1607.01759

2 数据介绍

本文使用的数据集为网站爬取的短文本评论，其中由于数据量较少，不区分验证集与测试集，并按照4：1的比例将原始有标签数据分为训练集与测试集。
原始数据文件压缩包网盘链接：https://pan.baidu.com/s/13vwd3lfKWfXlD1a8uB6ngg
提取码：urj2
注：解压后的text_data文件夹放置在与程序的同级目录下。
原始数据以txt文本格式保存，标签分为pos与neg，因此预处理主要是将标签与数据分开对应保存，并将数据分为训练集与测试集。

3 数据预处理

import os
# 获取text_data文件夹下的所有文件路径
temp_list = list(os.walk(r"text_data"))
original = temp_list[0][0]
file_name = temp_list[0][2]
path_list = [original + "\\" + eve_name for eve_name in file_name]
# 创建所需文件
train_data = open(r"train_data.txt", "w", encoding="utf-8")
train_label = open(r"train_label.txt", "w", encoding="utf-8")
test_data = open(r"test_data.txt", "w", encoding="utf-8")
test_label = open(r"test_label.txt", "w", encoding="utf-8")
vocabulary = open(r"vocabulary.txt", "w", encoding="utf-8")
# 将原始数据进行标签分离与训练测试集分离
for every_path in path_list:
    with open(every_path, "r", encoding="utf-8") as temp_file:
        corpus = [eve for eve in temp_file if len(eve.strip("\n")) != 0]
        limit1 = len(corpus)*0.9
        limit2 = len(corpus)*0.1
        for i in range(len(corpus)):
            if limit2 < i < limit1:
                if corpus[i][:3] == "pos":
                    train_data.write(corpus[i][3:])
                    train_label.write("1" + "\n")
                else:
                    train_data.write(corpus[i][3:])
                    train_label.write("0" + "\n")
            else:
                if corpus[i][:3] == "pos":
                    test_data.write(corpus[i][3:])
                    test_label.write("1" + "\n")
                else:
                    test_data.write(corpus[i][3:])
                    test_label.write("0" + "\n")
# 创建字库vocabulary_2gram，包含原始数据中所有的字，写入vocabulary.txt待用
from nltk.util import ngrams
with open(r"test_data.txt", "r", encoding="utf-8") as file1:
	corpus1 = [eve for eve in file1]
with open(r"train_data.txt", "r", encoding="utf-8") as file2:
	corpus2 = [eve for eve in file2]
with open(r"vocabulary_2gram.txt","w",encoding="utf-8") as file3:
	word_list = []
    corpus = corpus1 + corpus2
     for line in corpus:
         word_list.append([char for char in line])
     _2gramword_list = []
     for eve in word_list:
         temp = ngrams(eve,2)
         for turple in temp:
             _2gramword_list.append(turple[0]+turple[1])
     word_list = list(set(_2gramword_list))
     for word in word_list:
         file3.write(word + "\n")

4 fasttext实现文本分类

from nltk.util import ngrams
import tensorflow.contrib.keras as kr
import tensorflow as tf

with open(r"train_data.txt", "r", encoding="utf-8") as file1:
    corpus = [eve.strip("\n") for eve in file1]
with open(r"vocabulary_2gram.txt", "r", encoding="utf-8") as file2:
    vocabulary = [word.strip("\n") for word in file2]
with open(r"train_label.txt", "r", encoding="utf-8") as file3:
    label_list = [int(eve.strip("\n")) for eve in file3]
assert len(label_list) == len(corpus)

word2id = {word:id_ for id_, word in enumerate(vocabulary)}
def line2id_2gram(line):
    temp = []
    for char in line:
        temp.append(char)
    tep = [eve[0] + eve[1] for eve in ngrams(temp,2)]
    return [word2id[word] for word in tep]
train_list = [line2id_2gram(line) for line in corpus]
train_x = kr.preprocessing.sequence.pad_sequences(train_list, 100)  # 长度一致train_x
train_y = kr.utils.to_categorical(label_list, num_classes=2)
tf.compat.v1.reset_default_graph()
X_holder = tf.compat.v1.placeholder(tf.int32, [None, 100])  # 占位
Y_holder = tf.compat.v1.placeholder(tf.float32, [None, 2])

# 做词嵌入工作 注意71166是自由生成的行向量，这里是构建的vocabulary_2gram.txt中的大小
embedding = tf.compat.v1.get_variable('embedding', [71166, 60])  # 一种初始化变量的方法，随机初始化了矩阵变量
embedding_inputs = tf.nn.embedding_lookup(embedding, X_holder)  # lookup


# 神经网络结构 输入-取平均-softmax二分类器-输出
mean = tf.reduce_mean(embedding_inputs, axis=1)  # 将句子中的字按照字向量取平均值
logits = tf.layers.dense(mean, 2)  # 接一个60：2的softmax的分类器

learning_rate = tf.train.polynomial_decay(1e-2, 0, 1)  # rate = (rate - 0.0001) *(1 - 0 / 1) ^ (1) +0.0001

cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=Y_holder, logits=logits)
loss = tf.reduce_mean(cross_entropy)  # 熵的平均值
optimizer = tf.train.AdamOptimizer(learning_rate)  # 定义优化器
train = optimizer.minimize(loss)  # 将优化器与损失值连接起来

isCorrect = tf.equal(tf.argmax(Y_holder, 1), tf.argmax(logits, 1))  # 判断是否正确
accuracy = tf.reduce_mean(tf.cast(isCorrect, tf.float32))  # 判断准确率

init = tf.global_variables_initializer()
session = tf.Session()
session.run(init)

with open(r"test_data.txt", "r", encoding="utf-8") as file4:
    corpus_ = [eve.strip("\n") for eve in file4]
with open(r"test_label.txt", "r", encoding="utf-8") as file5:
    label_list_ = [int(eve.strip("\n")) for eve in file5]
assert len(label_list_) == len(corpus_)
test_list = [line2id_2gram(line) for line in corpus_]
test_x = kr.preprocessing.sequence.pad_sequences(test_list, 100)  # 长度一致train_x
test_y = kr.utils.to_categorical(label_list_, num_classes=2)

import random
for i in range(3000):
    selected_index = random.sample(list(range(len(train_y))), k=60)  # 批训练大小的意思就是多少个样本调整一次参数
    batch_X = train_x[selected_index]
    batch_Y = train_y[selected_index]
    session.run(train, {X_holder:batch_X, Y_holder:batch_Y})
    step = i + 1
    if step % 100 == 0:
        selected_index = random.sample(list(range(len(test_y))), k=150)
        batch_X = test_x[selected_index]
        batch_Y = test_y[selected_index]
        loss_value, accuracy_value = session.run([loss, accuracy], {X_holder:batch_X, Y_holder:batch_Y})
        print('step:%d loss:%.4f accuracy:%.4f' %(step, loss_value, accuracy_value))

5 运行结果

step:100 loss:0.2895 accuracy:0.8800
step:200 loss:0.2355 accuracy:0.8933
step:300 loss:0.1878 accuracy:0.9333
step:400 loss:0.3736 accuracy:0.8933
step:500 loss:0.2616 accuracy:0.8867
step:600 loss:0.2473 accuracy:0.9133
step:700 loss:0.1472 accuracy:0.9333
step:800 loss:0.3756 accuracy:0.8933
step:900 loss:0.2291 accuracy:0.9333
step:1000 loss:0.2153 accuracy:0.9200
step:1100 loss:0.2682 accuracy:0.9000
step:1200 loss:0.2792 accuracy:0.9133
step:1300 loss:0.2163 accuracy:0.9267
step:1400 loss:0.3535 accuracy:0.8867
step:1500 loss:0.2507 accuracy:0.9133
step:1600 loss:0.2644 accuracy:0.9000
step:1700 loss:0.4856 accuracy:0.8600
step:1800 loss:0.3144 accuracy:0.8800
step:1900 loss:0.2887 accuracy:0.8733
step:2000 loss:0.3268 accuracy:0.9067
step:2100 loss:0.3687 accuracy:0.8667
step:2200 loss:0.2661 accuracy:0.9267
step:2300 loss:0.3162 accuracy:0.9133
step:2400 loss:0.5905 accuracy:0.8733
step:2500 loss:0.3979 accuracy:0.8800
step:2600 loss:0.3052 accuracy:0.9333
step:2700 loss:0.3631 accuracy:0.9600
step:2800 loss:0.3845 accuracy:0.9200
step:2900 loss:0.5608 accuracy:0.8800
step:3000 loss:0.1405 accuracy:0.9400

结果准确度还可以，并且模型的运行速度非常快。根据作者原文的说法，fasttext比较适合解决样本量大，并且类标签种类比较多的数据集。另外，本文实现的fasttext实际上只实现了模型结构与Ngram特征提取，层次softmax并未实现，因为解决的问题不是多分类的问题。