用LSTM、GRU来训练字符级的语言模型
import torch
import torch.nn as nn
import torch.utils.data as Data
import torch.autograd as autograd
import torch.optim as optim
import numpy as np
from torch.autograd import Variable
poetrys = []
poetry = ''
with open("poetryFromTang.txt", encoding='utf-8') as f:
next(f)
for line in f:
if len(line)!=1:
poetry += line.strip('\n')
else:
poetrys.append(poetry)
poetry = ''
all_word = ''
for potery in poetrys:
all_word += potery
all_word = all_word.replace(',','').replace('。','')
word_dict = {
}
for word in all_word:
if word not in word_dict:
word_dict[word] = 1
else:
word_dict[word] += 1
word_sort = sorted(word_dict.items(),key=lambda x:x[1],reverse=True)
words, _ = zip(*word_sort)
word_to_token = {
word:id for id, word in enumerate(words)}
token_to_word = dict(enumerate(words))
def transword(char_list):
ids = [word_to_token.get(char, len(word)-1) for