Tensorflow学习笔记--使用keras完成文本情感分类问题

jilu

# -*- coding:utf-8 -*-
import numpy as np
import pandas as pd
import jieba
import csv

df = pd.read_csv('train_data.csv',encoding='utf-8')
df['label']=1
df.loc[df['satisfaction_id']==1,['label']]=1
df.loc[df['satisfaction_id']==3,['label']]=0
df['send_content_words'] = df['send_content'].apply(lambda s: list(jieba.cut(s))) #调用结巴分词

maxlen = 100 #截断词数
min_count = 5 #出现次数少于该值的词扔掉。这是最简单的降维方法

content = []
for i in df['send_content_words']:
	content.extend(i)
--diction
abc = pd.Series(content).value_counts()
abc = abc[abc >= min_count]

abc[:] = range(1, len(abc)+1)
abc[''] = 0

def doc2num(s, maxlen): 
    s = [i for i in s if i in abc.index]
    s = s[:maxlen] + ['']*max(0, maxlen-len(s))
    return list(abc[s])
df['doc2num'] = df['send_content_words'].apply(lambda s: doc2num(s, maxlen))

#手动打乱数据
idx = range(len(df))
np.random.shuffle(idx)
all_ = df.loc[idx]

#按keras的输入要求来生成数据
x = np.array(list(all_['doc2num']))
y = np.array(list(all_['label']))
y = y.reshape((-1,1)) #调整标签形状

from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Embedding
from keras.layers import LSTM,Bidirectional

#建立模型
model = Sequential()
model.add(Embedding(len(abc), 64, input_length=maxlen))
model.add(Bidirectional(LSTM(128)))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

batch_size = 128
train_num = 15000

model.fit(x, y, batch_size = batch_size, nb_epoch=30)

--test
df_test = pd.read_csv('test_data.csv',encoding='utf-8')
df_test['label']=1
df_test.loc[df['satisfaction_id']==1,['label']]=1
df_test.loc[df['satisfaction_id']==3,['label']]=0
df_test['send_content_words'] = df_test['send_content'].apply(lambd