有时,当没有limit for code generation或the or tokens are not numerical tokens时,LSTM永远不会收敛。如果可以发送输出或错误消息,那么调试起来就容易多了。在
你可以创建一个额外的类来获取单词和句子。在# tokens for start of sentence(SOS) and end of sentence(EOS)
SOS_token = 0
EOS_token = 1
class Lang:
'''
class for word object, storing sentences, words and word counts.
'''
def __init__(self, name):
self.name = name
self.word2index = {}
self.word2count = {}
self.index2word = {0: "SOS", 1: "EOS"}
self.n_words = 2 # Count SOS and EOS
def addSentence(self, sentence):
for word in sentence.split(' '):
self.addWord(word)
def addWord(self, word):
if word not in self.word2index:
self.word2index[word] = self.n_words
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1