机器学习项目4聊天机器人

聊天机器人

翻译链接 数据
聊天机器人模型分为基于检索的模型和基于生成的模型
基于检索的Chatbots 基于检索的聊天机器人使用预定义的输入模式和回应。它使用某种启发式方法来选择适当的回应。
基于生成的Chatbots 基于seq2seq 神经网络，将输入数据转为输出，需要大量数据。
在此项目中，用一个特别热递归神经网络LSTM首先判断用户的信息属于哪个类，然后在该类中随机选一个回应。共5步分别是：
1、 导入加载数据文件
2、 处理数据
3、 创建训练集和测试集
4、 创建、训练、保存模型
5、 预测与可视化
数据集截图

第一步： 导入和加载数据文件，数据存储格式为JSON,因此在python中需要用json包来解析JSON文件

# 自然语言处理库
import nltk
# 词形还原 如cars还原为car, eat还原为eat
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
import json
import pickle
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.optimizers import SGD
import random
words=[]
classes = []
documents = []
ignore_words = ['?', '!']
# 打开读取文件
data_file = open('intents.json').read()
# json.loads()将已编码的JSON字符串解码为 Python 对象
intents = json.loads(data_file)

第二步 处理数据，获得训练样本和其标签（将用户的输入句子和所属类别转为样本向量和标签向量）

# intents里面包含 intents
# intent里面包含tag, patterns, responses, context
for intent in intents['intents']:
    for pattern in intent['patterns']:
        #tokenize each word
        # 如报错无punkt，wordnet可在github下载该压缩包然后解压到程序搜索过的任一路径
        # 如C:\Users\user\anaconda3\envs\tensorflow\share\nltk_data\tokenizers\punkt
        # 分词，将句子段落分成为字词单位
        w = nltk.word_tokenize(pattern)
        words.extend(w)
        #add documents in the corpus
        # 将该词和其所在句子类别共同划进去
        documents.append((w, intent['tag']))
        # add to our classes list
        if intent['tag'] not in classes:
            classes.append(intent['tag'])

# lemmaztize and lower each word and remove duplicates
# 先将所有单词转为小写，然后词形还原,去掉重复词语
words = [lemmatizer.lemmatize(w.lower()) for w in words if w not in ignore_words]
# 排序a-z
words = sorted(list(set(words)))
# sort classes
classes = sorted(list(set(classes)))
# documents = combination between patterns and intents
print (len(documents), "documents")
# classes = intents
print (len(classes), "classes", classes)
# words = all words, vocabulary
print (len(words), "unique lemmatized words", words)
# pickle.dump(obj, file[, protocol]) 序列化对象，并将结果数据流写入到文件对象中
pickle.dump(words,open('words.pkl','wb'))
pickle.dump(classes,open('classes.pkl','wb'))
# create our training data
training = []
# create an empty array for our output
output_empty = [0] * len(classes)
# training set, bag of words for each sentence
# 生成训练集，样本及对应标签
for doc in documents:
    # initialize our bag of words
    bag = []
    # list of tokenized words for the pattern
    pattern_words = doc[0]
    # lemmatize each word - create base word, in attempt to represent related words
    pattern_words = [lemmatizer.lemmatize(word.lower()) for word in pattern_words]
    # create our bag of words array with 1, if word match found in current pattern
    # 词袋：将一个句子用装在一个袋子里的词来表示，不考虑顺序
    # 生成该句子对应的输入向量
    for w in words:
        bag.append(1) if w in pattern_words else bag.append(0)
    # 该句子对应的标签向量
    # output is a '0' for each tag and '1' for current tag (for each pattern)
    output_row = list(output_empty)
    output_row[classes.index(doc[1])] = 1
    training.append([bag, output_row])
# shuffle our features and turn into np.array
# 打乱
random.shuffle(training)
training = np.array(training)
# create train and test lists. X - patterns, Y - intents
# 样本和标签分开
train_x = list(training[:,0])
train_y = list(training[:,1])
print("Training data created")

第三步 创建模型，训练模型，保存模型

# Create model - 3 layers. First layer 128 neurons, second layer 64 neurons and 3rd output layer contains number of neurons
# equal to number of intents to predict output intent with softmax
# 创建模型
model = Sequential()
# 全连接层输出为128
model.add(Dense(128, input_shape=(len(train_x[0]),), activation='relu'))
# 随机丢弃一半的神经元
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
# 分类层
model.add(Dense(len(train_y[0]), activation='softmax'))
# 优化算法为随机梯度下降，学习率为0.01
# Compile model. Stochastic gradient descent with Nesterov accelerated gradient gives good results for this model
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
# 训练模型
#fitting and saving the model 
hist = model.fit(np.array(train_x), np.array(train_y), epochs=200, batch_size=5, verbose=1)
# 保存模型
model.save('chatbot_model.h5', hist)
print("model created")

第四步 实现交互窗口，调用训练好的模型，对用户输入进行预测

import nltk
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
import pickle
import numpy as np
from keras.models import load_model
# 加载模型
model = load_model('chatbot_model.h5')
import json
import random
# 解析json文件
intents = json.loads(open('intents.json').read())
# 加载词和类别
words = pickle.load(open('words.pkl','rb'))
classes = pickle.load(open('classes.pkl','rb'))
# 对用户输入句子进行分词，变小写，词形还原，去重
def clean_up_sentence(sentence):
    # tokenize the pattern - split words into array
    sentence_words = nltk.word_tokenize(sentence)
    # stem each word - create short form for word
    sentence_words = [lemmatizer.lemmatize(word.lower()) for word in sentence_words]
    return sentence_words
# 对处理的句子进行向量化
# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence
def bow(sentence, words, show_details=True):
    # tokenize the pattern
    # 调整大小写，词形还原，去重得到句子
    sentence_words = clean_up_sentence(sentence)
    # bag of words - matrix of N words, vocabulary matrix
    bag = [0]*len(words)
    # 向量化
    for s in sentence_words:
        for i,w in enumerate(words):
            if w == s: 
                # assign 1 if current word is in the vocabulary position
                bag[i] = 1
                if show_details:
                    print ("found in bag: %s" % w)
    return(np.array(bag))
# 对句子进行预测，看所属类别
def predict_class(sentence, model):
    # filter out predictions below a threshold
    # 句子向量化
    p = bow(sentence, words,show_details=False)
    # 预测
    res = model.predict(np.array([p]))[0]
    ERROR_THRESHOLD = 0.25
    # 保存概率大于0.25的索引，即类别
    results = [[i,r] for i,r in enumerate(res) if r>ERROR_THRESHOLD]
    # sort by strength of probability
    # 排序
    results.sort(key=lambda x: x[1], reverse=True)
    return_list = []
    for r in results:
        return_list.append({"intent": classes[r[0]], "probability": str(r[1])})
    return return_list
# 随机返回句子对应类别中预定义回答
def getResponse(ints, intents_json):
    tag = ints[0]['intent']
    list_of_intents = intents_json['intents']
    for i in list_of_intents:
        if(i['tag']== tag):
            result = random.choice(i['responses'])
            break
    return result

def chatbot_response(msg):
    # 调用预测函数
    ints = predict_class(msg, model)
    # 输入预测类ints,在相同类中随机选一个句子返回
    res = getResponse(ints, intents)
    return res

#Creating GUI with tkinter
import tkinter
from tkinter import *
def send():
    # text .get(start, end)获取输入框的值
    # 文本框控件中第一个字符的位置是 1.0， end-1c输入结束之前的一个位置，去掉换行符
    # .strip()去掉字符串的首尾指定字符默认为空格和换行
    msg = EntryBox.get("1.0",'end-1c').strip()
    EntryBox.delete("0.0",END)

    if msg != '':
        # text控件配置normal表示用户可以编辑、添加、插入或编辑文本内容
        ChatLog.config(state=NORMAL)
        # 在最后插入
        ChatLog.insert(END, "You: " + msg + '\n\n')
        ChatLog.config(foreground="#442265", font=("Verdana", 12 ))
        # 调用函数得到返回值
        res = chatbot_response(msg)
        ChatLog.insert(END, "Bot: " + res + '\n\n')
            
        ChatLog.config(state=DISABLED)
        ChatLog.yview(END)
 

base = Tk()
# 窗口名字
base.title("Hello")
# 窗口大小
base.geometry("400x500")
base.resizable(width=FALSE, height=FALSE)

#Create Chat window
# Text文本组件用于显示和处理多行文本
ChatLog = Text(base, bd=0, bg="white", height="8", width="50", font="Arial",)
# disabled只读
ChatLog.config(state=DISABLED)
# scrollbar一个滑动控制器，用于实现垂直滚动小部件，如Listbox、Text和Canvas。
# 还可以在Entry小部件上创建水平滚动条。
#Bind scrollbar to Chat window
# 控件和滚动条结合的步骤
# 1.将这些控件的yscrollcommand选项设置为scrollbar的set方法。
# 2.将scrollbar的command选项设置为这些控件的yview方法。
scrollbar = Scrollbar(base, command=ChatLog.yview, cursor="heart")
# yscrollcommand在y轴使用滚动条
ChatLog['yscrollcommand'] = scrollbar.set

# send函数触发
#Create Button to send message
SendButton = Button(base, font=("Verdana",12,'bold'), text="Send", width="12", height=5,
                    bd=0, bg="#32de97", activebackground="#3c9d9b",fg='#ffffff',
                    command= send )
# 输入框创建
#Create the box to enter message
EntryBox = Text(base, bd=0, bg="white",width="29", height="5", font="Arial")
#EntryBox.bind("<Return>", send)

# 放置所有空间
#Place all components on the screen
scrollbar.place(x=376,y=6, height=386)
ChatLog.place(x=6,y=6, height=386, width=370)
EntryBox.place(x=128, y=401, height=90, width=265)
SendButton.place(x=6, y=401, height=90)

base.mainloop()