1.数据预处理与模型训练
''' 数据集介绍
中文酒店评论,7766笔数据,分为正面、负面评价
'''
import paddle
import paddle.dataset.imdb as imdb
import paddle.fluid as fluid
import numpy as np
import os
import random
from multiprocessing import cpu_count
mydict = {
}
code = 1
data_file = "data/hotel_discuss2.csv"
dict_file = "data/hotel_dict.txt"
encoding_file = "data/hotel_encoding.txt"
puncts = " \n"
with open(data_file, "r", encoding="utf-8-sig") as f:
for line in f.readlines():
trim_line = line.strip()
for ch in trim_line:
if ch in puncts:
continue
if ch in mydict:
continue
elif len(ch) <= 0:
continue
else:
mydict[ch] = code
code += 1
code += 1
mydict["<unk>"] = code
with open(dict_file, "w", encoding="utf-8-sig") as f:
f.write(str(mydict))
print("数据字典保存完成!")
def load_dict():
with open(dict_file, "r", encoding="utf-8-sig") as f:
lines = f.readlines()
new_dict = eval(lines[0])
return new_dict
new_dict = load_dict()
with open(data_file, "r", encoding="utf-8-sig") as f:
with open(encoding_file, "w", encoding="utf-8-sig") as fw:
for line in f.readlines()