Github代码链接：https://github.com/nyu-dl/dl4mt-tutorial/blob/master/session0/lm.py如有理解错误，还望能够指出；# -*- coding: utf-8 -*- ''' Build a simple neural language model using GRU units ''' import theano import theano.tensor as tensor from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams import cPickle as pkl import ipdb import numpy import copy import os import warnings import sys import time from collections import OrderedDict from data_iterator import TextIterator profile = False # push parameters to Theano shared variables def zipp(params, tparams): for kk, vv in params.iteritems(): tparams[kk].set_value(vv) # pull parameters from Theano shared variables def unzip(zipped): new_params = OrderedDict() for kk, vv in zipped.iteritems(): new_params[kk] = vv.get_value() return new_params # get the list of parameters: Note that tparams must be OrderedDict def itemlist(tparams): return [vv for kk, vv in tparams.iteritems()] # dropout def dropout_layer(state_before, use_noise, trng): proj = tensor.switch( use_noise, state_before * trng.binomial(state_before.shape, p=0.5, n=1, dtype=state_before.dtype), state_before * 0.5) return proj # make prefix-appended name def _p(pp, name): return '%s_%s' % (pp, name) # initialize Theano shared variables according to the initial parameters，所有变量共享 def init_tparams(params): tparams = OrderedDict() for kk, pp in params.iteritems(): tparams[kk] = theano.shared(params[kk], name=kk) return tparams # load parameters def load_params(path, params): pp = numpy.load(path) for kk, vv in params.iteritems(): if kk not in pp: warnings.warn('%s is not in the archive' % kk) continue params[kk] = pp[kk] return params # layers: 'name': ('parameter initializer', 'feedforward') layers = {'ff': ('param_init_fflayer', 'fflayer'), 'gru': ('param_init_gru', 'gru_layer'), } def get_layer(name): fns = layers[name] return (eval(fns[0]), eval(fns[1])) # orthogonal initialization for weights # see Saxe et al. ICLR'14 # 根据奇异值分解来求解特征 def ortho_weight(ndim): W = numpy.random.randn(ndim, ndim) u, s, v = numpy.linalg.svd(W) return u.astype('float32') # weight initializer, normal by default #nin = n_words,词表的大小，nout =1000，词向量的维度 def norm_weight(nin, nout=None, scale=0.01, ortho=True): if nout is None: nout = nin if nout == nin and ortho: #如果 词向量维度没有给，所以用到奇异值分解，得到embedding 矩阵 W = ortho_weight(nin) else: #否则的话，根据给定的词向量的维度，初始化embedding 矩阵 W = scale * numpy.random.randn(nin, nout) return W.astype('float32') def tanh(x): return tensor.tanh(x) def linear(x): return x def concatenate(tensor_list, axis=0): """ Alternative implementation of `theano.tensor.concatenate`. This function does exactly the same thing, but contrary to Theano's own implementation, the gradient is implemented on the GPU. Backpropagating through `theano.tensor.concatenate` yields slowdowns because the inverse operation (splitting) needs to be done on the CPU. This implementation does not have that problem. :usage: >>> x, y = theano.tensor.matrices('x', 'y') >>> c = concatenate([x, y], axis=1) :parameters: - tensor_list : list list of Theano tensor expressions that should be concatenated. - axis : int the tensors will be joined along this axis. :returns: - out : tensor the concatenated tensor expression. """ concat_size = sum(tt.shape[axis] for tt in tensor_list) output_shape = () for k in range(axis): output_shape += (tensor_list[0].shape[k],) output_shape += (concat_size,) for k in range(axis + 1, tensor_list[0].ndim): output_shape += (tensor_list[0].shape[k],) out = tensor.zeros(output_shape) offset = 0 for tt in tensor_list: indices = () for k in range(axis): indices += (slice(None),) indices += (slice(offset, offset + tt.shape[axis]),) for k in range(axis + 1, tensor_list[0].ndim): indices += (slice(None),) out = tensor.set_subtensor(out[indices], tt) offset += tt.shape[axis] return out # batch preparation, returns padded batch and mask def prepare_data(seqs_x, maxlen=None, n_words=30000): # x: a list of sentences lengths_x = [len(s) for s in seqs_x] # filter according to mexlen if maxlen is not None: new_seqs_x = [] new_lengths_x = [] for l_x, s_x in zip(lengths_x, seqs_x): if l_x < maxlen: new_seqs_x.append(s_x) new_lengths_x.append(l_x) lengths_x = new_lengths_x seqs_x = new_seqs_x if len(lengths_x) < 1: return None, None, None, None n_samples = len(seqs_x) maxlen_x = numpy.max(lengths_x) + 1 x = numpy.zeros((maxlen_x, n_samples)).astype('int64') x_mask = numpy.zeros((maxlen_x, n_samples)).astype('float32') ''' /*这里，将原来的seqs中的每一行加入到新的x的相应的列中，如果原来的这行的长度（也就是step） 小于maxlen（新矩阵x的行数，也是原来step最长的样本的step， 那么seqs的那行转换成的x的列不足的部分就是0，相当于seqs中的行转置了一下，变成新的x的一列， 长度不够后边就补0，而mask就记录了新的x矩阵每个位置是否有值，有值得话mask相应的位置就是1，没有的话就是0）*/ ''' for idx, s_x in enumerate(seqs_x): x[:lengths_x[idx], idx] = s_x x_mask[:lengths_x[idx]+1, idx] = 1. return x, x_mask # feedforward layer: affine transformation + point-wise nonlinearity #初始化每一层的W和b def param_init_fflayer(options, params, prefix='ff', nin=None, nout=None, ortho=True): if nin is None: nin = options['dim_proj'] if nout is None: nout = options['dim_proj'] params[_p(prefix, 'W')] = norm_weight(nin, nout, scale=0.01, ortho=ortho) params[_p(prefix, 'b')] = numpy.zeros((nout,)).astype('float32') return params def fflayer(tparams, state_below, options, prefix='rconv', activ='lambda x: tensor.tanh(x)', **kwargs): return eval(activ)( tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]) # GRU layer def param_init_gru(options, params, prefix='gru', nin=None, dim=None): #dim_proj / *word,embedding的维数和隐藏层的维数，用默认值。（word embedding是一种将一个词转成一个向量的过程，这里不去深究） * / #nin = dim_word = 512, n_out = dim = 1024 if nin is None: nin = options['dim_proj'] if dim is None: dim = options['dim_proj'] # embedding to gates transformation weights, biases ''' # concatenate 函数 a = [[1,2],[3,4]] b = [[5,6],[7,8]] numpy.concatenate([a,b],axis=1) array([[1, 2, 5, 6], [3, 4, 7, 8]]) np.concatenate([aa,bb],axis=0) array([[1, 2], [3, 4], [5, 6], [7, 8]]) ''' #W.shape = nin*(nin*dim) 512 * (512*1024) W = numpy.concatenate([norm_weight(nin, dim), norm_weight(nin, dim)], axis=1) params[_p(prefix, 'W')] = W #b.shape 2*dim 2048 params[_p(prefix, 'b')] = numpy.zeros((2 * dim,)).astype('float32') # recurrent transformation weights for gates #ortho_weight(dim),shape = dim * dim 1024 * (1024*1024) U = numpy.concatenate([ortho_weight(dim), ortho_weight(dim)], axis=1) params[_p(prefix, 'U')] = U # embedding to hidden state proposal weights, biases #Wx.shaope = dim_word*dim 512 * 1024 Wx = norm_weight(nin, dim) params[_p(prefix, 'Wx')] = Wx params[_p(prefix, 'bx')] = numpy.zeros((dim,)).astype('float32') # recurrent transformation weights for hidden state proposal # Ux.shape = dim * dim Ux = ortho_weight(dim) params[_p(prefix, 'Ux')] = Ux return params def gru_layer(tparams, state_below, options, prefix='gru', mask=None, one_step=False, init_state=None, **kwargs): if one_step: assert init_state, 'previous state must be provided' nsteps = state_below.shape[0] #//state_below是输入x和w，b计算后的输入节点。同上，第一维代表step #第一维度表示的是nsteps，为迭代次数 #第二维度表示样本数，如果输入三维的x，那么样本数就是第二维的长度，否则就是只有一个样本 if state_below.ndim == 3: n_samples = state_below.shape[1] else: n_samples = state_below.shape[0] dim = tparams[_p(prefix, 'Ux')].shape[1] # if mask is None: mask = tensor.alloc(1., state_below.shape[0], 1) # utility function to slice a tensor #// 切片，计算的时候是几个门一起计算，切片将各个门的值分开 def _slice(_x, n, dim): if _x.ndim == 3: return _x[:, :, n*dim:(n+1)*dim] return _x[:, n*dim:(n+1)*dim] # state_below is the input word embeddings # input to the gates, concatenated state_below_ = tensor.dot(state_below, tparams[_p(prefix, 'W')]) + \ tparams[_p(prefix, 'b')] # input to compute the hidden state proposal state_belowx = tensor.dot(state_below, tparams[_p(prefix, 'Wx')]) + \ tparams[_p(prefix, 'bx')] # step function to be used by scan # arguments | sequences |outputs-info| non-seqs ''' #/*scan函数，进行迭代的函数是_step，它的输入是m_, x_, h_, c_，迭代的是sequences中的mask，state_below,每次拿出他们的一行， # 作为输入的m_和x_，h_和c_的初始值设置为0（outputs_info设置初始值）， # 每次计算，_step返回的h和c分别赋给下次迭代的h_和c_，迭代次数为nsteps， # 这样就实现了隐藏层节点的传递，最后函数返回h和c给rval。*/ zt=sigmoid(Wzxt+Uzht−1) rt=sigmoid(Wtxt+Utht−1) h˜t=tanh(Wxt+U(rt∘ht−1)) ht=(1−zt)∘h(t−1)+zt∘h˜t ''' def _step_slice(m_, x_, xx_, h_,U, Ux): #x_ = state_below_ = emb*encoder_w+encoder_b #preact = h(t-1) * U + x_ preact = tensor.dot(h_, U) preact += x_ # reset and update gates #此处“r”和“u”同时计算 # u = zt = sigmoid(Wzxt + Uzht−1) # r = rt = sigmoid(Wtxt + Utht−1) r = tensor.nnet.sigmoid(_slice(preact, 0, dim)) # rt u = tensor.nnet.sigmoid(_slice(preact, 1, dim)) # zt # compute the hidden state proposal #xx_ = state_below_x = Wxt = emb*encoder_wx+encoder_bx #GRU公式：h = h˜t=tanh(Wxt+U(r(t)∘h(t−1))) preactx = tensor.dot(h_, Ux) preactx = preactx * r preactx = preactx + xx_ # hidden state proposal h = tensor.tanh(preactx) #GRU 公式：ht=(1−zt)∘h(t−1)+zt∘h˜t # leaky integrate and obtain next hidden state #如果样本到一定的状态后没有值了，也就是mask矩阵对应位置的值为0了，那么它的好就和上一个时刻的保持不变。 h = u * h_ + (1. - u) * h h = m_[:, None] * h + (1. - m_)[:, None] * h_ return h # prepare scan arguments seqs = [mask, state_below_, state_belowx] _step = _step_slice shared_vars = [tparams[_p(prefix, 'U')], tparams[_p(prefix, 'Ux')]] # set initial state to all zeros if init_state is None: #theano.alloc create a tensor shape is (n_samples,dim) init_state = tensor.unbroadcast(tensor.alloc(0., n_samples, dim), 0) ''' /*scan函数，进行迭代的函数是_step，它的输入是m_, x_, xx_, h_， 迭代的是sequences中的mask，state_below,每次拿出他们的一行， 作为输入的m_和x_，xx_和h_的初始值设置为0（outputs_info设置初始值）， 每次计算，_step返回的h分别赋给下次迭代的h_，迭代次数为nsteps， 这样就实现了隐藏层节点的传递，最后函数返回h给rval。*/ ''' if one_step: # sampling rval = _step(*(seqs+[init_state]+shared_vars)) else: # training rval, updates = theano.scan(_step, sequences=seqs, outputs_info=[init_state], non_sequences=shared_vars, name=_p(prefix, '_layers'), n_steps=nsteps, profile=profile, strict=True) rval = [rval] return rval # initialize all parameters def init_params(options): params = OrderedDict() # embedding,create embedding vector ''' dim_word:词向量的维度，n_word:词向量的大小，dim：1024 /* 随机生成embedding矩阵，这里为30000 * 512维的，因为词典大小是30000，也就是说，词的ID范围是1-30000， 我们将每个词转换成一个512维的向量，所以这里生成了一个10000*512的矩阵，每个词转换成它的ID的那一行的512维向量。 比如“我”这个词的ID是5，那么“我”这个词就用params['Wemb']矩阵的第5行表示， 第5行就是一个512维的向量，这里用随机生成的矩阵表示，作为示例。（这是下边用到的，这里先给出解释）*/ ''' params['Wemb'] = norm_weight(options['n_words'], options['dim_word']) ''' layers = {'ff': ('param_init_fflayer', 'fflayer'), 'gru': ('param_init_gru', 'gru_layer'), } ''' ''' embedding = 30000*512 n_words*dim_word encoder = 512*1024 dim_word*dim ff_logit_lstm= 1024*512 dim*dim_word ff_logit_prev=512*512 dim_word*dim_word ff_logit = 512*30000 dim_word*n_words ''' # options['encoder'] = "GRU",get_layer(options['encoder']) = 'gru': ('param_init_gru', 'gru_layer') #调用函数为"param_init_gru' nin = dim_word = 512, n_out = dim = 1024, prefix 为初始化的权值类型 #调用gru layer，param_init_gru, return W, U, Wx ,Ux params = get_layer(options['encoder'])[0](options, params, prefix='encoder', nin=options['dim_word'], dim=options['dim']) # readout #调用 param_init_fflayer，初始化相应的w和b params = get_layer('ff')[0](options, params, prefix='ff_logit_lstm', nin=options['dim'], nout=options['dim_word'], ortho=False) params = get_layer('ff')[0](options, params, prefix='ff_logit_prev', nin=options['dim_word'], nout=options['dim_word'], ortho=False) params = get_layer('ff')[0](options, params, prefix='ff_logit', nin=options['dim_word'], nout=options['n_words']) return params # build a training model def build_model(tparams, options): opt_ret = dict() trng = RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) # description string: #words x #samples x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') n_timesteps = x.shape[0] n_samples = x.shape[1] # input #tparms 此时是共享变量，type is OrderedDict() #dim_word = 512 #batch,step,change it to three dim emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) emb_shifted = tensor.zeros_like(emb) emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1]) emb = emb_shifted opt_ret['emb'] = emb # pass through gru layer, recurrence here #计算出来隐藏层 #gru_layer proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder', mask=x_mask) #proj_h = rval, updates = theano.scan(_step,sequences=seqs,outputs_info=[init_state],non_sequences=shared_vars,name=_p(prefix, '_layers'),n_steps=nsteps,profile=profile,strict=True) #get the ans is h proj_h = proj[0] opt_ret['proj_h'] = proj_h # compute word probabilities,计算词的概率 #调用 get_layer('ff')[1] = fflayer # logit_lstm = proj_h*ff_logit_lstm_w +b logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear') #logit_prev = emb * ff_logit_prev_w + b logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit = tensor.tanh(logit_lstm+logit_prev) # logit = logit * ff_logit_w + b logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') logit_shp = logit.shape probs = tensor.nnet.softmax( logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])) # cost,calculate the cost x_flat = x.flatten() x_flat_idx = tensor.arange(x_flat.shape[0]) * options['n_words'] + x_flat cost = -tensor.log(probs.flatten()[x_flat_idx]) cost = cost.reshape([x.shape[0], x.shape[1]]) opt_ret['cost_per_sample'] = cost cost = (cost * x_mask).sum(0) return trng, use_noise, x, x_mask, opt_ret, cost # build a sampler def build_sampler(tparams, options, trng): # x: 1 x 1 y = tensor.vector('y_sampler', dtype='int64') init_state = tensor.matrix('init_state', dtype='float32') # if it's the first word, emb should be all zero emb = tensor.switch(y[:, None] < 0, tensor.alloc(0., 1, tparams['Wemb'].shape[1]), tparams['Wemb'][y]) # apply one step of gru layer，line 278 proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder', mask=None, one_step=True, init_state=init_state) next_state = proj[0] # compute the output probability dist and sample logit_lstm = get_layer('ff')[1](tparams, next_state, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit = tensor.tanh(logit_lstm+logit_prev) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') next_probs = tensor.nnet.softmax(logit) next_sample = trng.multinomial(pvals=next_probs).argmax(1) # next word probability print 'Building f_next..', inps = [y, init_state] outs = [next_probs, next_sample, next_state] f_next = theano.function(inps, outs, name='f_next', profile=profile) print 'Done' return f_next # generate sample def gen_sample(tparams, f_next, options, trng=None, maxlen=30, argmax=False): sample = [] sample_score = 0 # initial token is indicated by a -1 and initial state is zero next_w = -1 * numpy.ones((1,)).astype('int64') next_state = numpy.zeros((1, options['dim'])).astype('float32') for ii in xrange(maxlen): inps = [next_w, next_state] ret = f_next(*inps) next_p, next_w, next_state = ret[0], ret[1], ret[2] if argmax: nw = next_p[0].argmax() else: nw = next_w[0] sample.append(nw) sample_score += next_p[0, nw] if nw == 0: break return sample, sample_score # calculate the log probablities on a given corpus using language model #用模型来计算给定的数据 def pred_probs(f_log_probs, prepare_data, options, iterator, verbose=True): probs = [] n_done = 0 for x in iterator: n_done += len(x) x, x_mask = prepare_data(x, n_words=options['n_words']) pprobs = f_log_probs(x, x_mask) for pp in pprobs: probs.append(pp) if numpy.isnan(numpy.mean(probs)): ipdb.set_trace() if verbose: print >>sys.stderr, '%d samples computed' % (n_done) return numpy.array(probs) # optimizers # name(hyperp, tparams, grads, inputs (list), cost) = f_grad_shared, f_update def adam(lr, tparams, grads, inp, cost, beta1=0.9, beta2=0.999, e=1e-8): gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k) for k, p in tparams.iteritems()] gsup = [(gs, g) for gs, g in zip(gshared, grads)] f_grad_shared = theano.function(inp, cost, updates=gsup, profile=profile) updates = [] t_prev = theano.shared(numpy.float32(0.)) t = t_prev + 1. lr_t = lr * tensor.sqrt(1. - beta2**t) / (1. - beta1**t) for p, g in zip(tparams.values(), gshared): m = theano.shared(p.get_value() * 0., p.name + '_mean') v = theano.shared(p.get_value() * 0., p.name + '_variance') m_t = beta1 * m + (1. - beta1) * g v_t = beta2 * v + (1. - beta2) * g**2 step = lr_t * m_t / (tensor.sqrt(v_t) + e) p_t = p - step updates.append((m, m_t)) updates.append((v, v_t)) updates.append((p, p_t)) updates.append((t_prev, t)) f_update = theano.function([lr], [], updates=updates, on_unused_input='ignore', profile=profile) return f_grad_shared, f_update def adadelta(lr, tparams, grads, inp, cost): zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] running_up2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rup2' % k) for k, p in tparams.iteritems()] running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function(inp, cost, updates=zgup+rg2up, profile=profile) updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg for zg, ru2, rg2 in zip(zipped_grads, running_up2, running_grads2)] ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) for ru2, ud in zip(running_up2, updir)] param_up = [(p, p + ud) for p, ud in zip(itemlist(tparams), updir)] f_update = theano.function([lr], [], updates=ru2up+param_up, on_unused_input='ignore', profile=profile) return f_grad_shared, f_update def rmsprop(lr, tparams, grads, inp, cost): zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] running_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad' % k) for k, p in tparams.iteritems()] running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function(inp, cost, updates=zgup+rgup+rg2up, profile=profile) updir = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_updir' % k) for k, p in tparams.iteritems()] updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] param_up = [(p, p + udn[1]) for p, udn in zip(itemlist(tparams), updir_new)] f_update = theano.function([lr], [], updates=updir_new+param_up, on_unused_input='ignore', profile=profile) return f_grad_shared, f_update def sgd(lr, tparams, grads, x, mask, y, cost): # allocate gradients and set them all to zero gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k) for k, p in tparams.iteritems()] # create gradient copying list, # from grads (tensor variable) to gshared (shared variable) gsup = [(gs, g) for gs, g in zip(gshared, grads)] # compile theano function to compute cost and copy gradients f_grad_shared = theano.function([x, mask, y], cost, updates=gsup, profile=profile) # define the update step rule pup = [(p, p - lr * g) for p, g in zip(itemlist(tparams), gshared)] # compile a function for update f_update = theano.function([lr], [], updates=pup, profile=profile) return f_grad_shared, f_update def train(dim_word=100, # word vector dimensionality dim=1000, # the number of GRU units encoder='gru', #/*一个标识符，encode端代码/ patience=10, # early stopping patience/*该参数用于earlystop，如果10轮迭代的误差没有降低，就进行earlystop*/ max_epochs=5000, finish_after=10000000, # finish after this many updates dispFreq=100, #/*每更新10次显示训练过程，即显示训练、验证和测试误差*/ decay_c=0., # L2 weight decay penalty /*参数U的正则权重，U为隐藏层ht到输出层的参数*/ lrate=0.01, n_words=100000, # vocabulary size，/*词典大小，用于数据预处理部分，将词用该词在词典中的ID表示，超过10000的用1表示，仅仅用于数据，不做深究*/ maxlen=100, # maximum length of the description optimizer='rmsprop', #/*优化方法，代码提供了sgd,adadelta和rmsprop三种方法，采用了adadelta*/ batch_size=16, #/*训练集用的*batch大小*/ valid_batch_size=16, #/*验证集用的*batch大小*/ saveto='model.npz', #/*保存最好模型的文件，保存训练误差，验证误差和测试误差等等*/ validFreq=1000, #/*验证频率*/ saveFreq=1000, # save the parameters after every saveFreq updates sampleFreq=100, # generate some samples after every sampleFreq dataset='/data/lisatmp3/chokyun/wikipedia/extracted/wiki.tok.txt.gz', valid_dataset='../data/dev/newstest2011.en.tok', dictionary='/data/lisatmp3/chokyun/wikipedia/extracted/' 'wiki.tok.txt.gz.pkl', use_dropout=False, reload_=False): # Model options #首先将当前的函数的局部作用，例如（use_dropout=False，转换为，model_options["user_dropout"] = False) #后面的很多参数就以model_options作为参数进行参数传递 model_options = locals().copy() # load dictionary，导入字典数据 with open(dictionary, 'rb') as f: worddicts = pkl.load(f) # invert dictionary，导入词典数据，具体格式不清楚，---------------------------------------------？？？？？？？ worddicts_r = dict() for kk, vv in worddicts.iteritems(): worddicts_r[vv] = kk # reload options ,如果有已经配置的参数的话，导入已经配置好的参数 if reload_ and os.path.exists(saveto): with open('%s.pkl' % saveto, 'rb') as f: model_options = pkl.load(f) print 'Loading data' # 数据迭代器 train = TextIterator(dataset, dictionary, n_words_source=n_words, batch_size=batch_size, maxlen=maxlen) valid = TextIterator(valid_dataset, dictionary, n_words_source=n_words, batch_size=valid_batch_size, maxlen=maxlen) print 'Building model' #搭建模型，传递过去所有参数， params = init_params(model_options) # reload parameters，如果已经有初始化完的数据，直接导入进来 if reload_ and os.path.exists(saveto): params = load_params(saveto, params) # create shared variables for parameters，使params的变量复制变为共享的变量 tparams = init_tparams(params) # build the symbolic computational graph #搭建模型，包括GRU计算和cost的计算 trng, use_noise, \ x, x_mask, \ opt_ret, \ cost = \ build_model(tparams, model_options) inps = [x, x_mask] print 'Buliding sampler' f_next = build_sampler(tparams, model_options, trng) # before any regularizer print 'Building f_log_probs...', f_log_probs = theano.function(inps, cost, profile=profile) print 'Done' cost = cost.mean() # apply L2 regularization on weights if decay_c > 0.: decay_c = theano.shared(numpy.float32(decay_c), name='decay_c') weight_decay = 0. for kk, vv in tparams.iteritems(): weight_decay += (vv ** 2).sum() weight_decay *= decay_c cost += weight_decay # after any regularizer - compile the computational graph for cost print 'Building f_cost...', f_cost = theano.function(inps, cost, profile=profile) print 'Done' print 'Computing gradient...', grads = tensor.grad(cost, wrt=itemlist(tparams)) print 'Done' # compile the optimizer, the actual computational graph is compiled here lr = tensor.scalar(name='lr') print 'Building optimizers...', #优化方式，rmsprop #梯度下降，进行参数的更新 f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, inps, cost) print 'Done' print 'Optimization' history_errs = [] # reload history if reload_ and os.path.exists(saveto): history_errs = list(numpy.load(saveto)['history_errs']) best_p = None bad_count = 0 # 记录误差、最好的结果，和bad_count计数 if validFreq == -1: validFreq = len(train[0])/batch_size if saveFreq == -1: saveFreq = len(train[0])/batch_size if sampleFreq == -1: sampleFreq = len(train[0])/batch_size #/*如果未设置验证频率和保存频率，那么就设置为一个epoch，len（train[0]）/batch_size就是一个epoch*/ # Training loop uidx = 0 estop = False bad_counter = 0 for eidx in xrange(max_epochs): n_samples = 0 for x in train: n_samples += len(x) uidx += 1 use_noise.set_value(1.) # pad batch and create mask #按照列来转化数据，每一列为一句话，x_mask是标记，这个位置有词，标记为1，反之为0. x, x_mask = prepare_data(x, maxlen=maxlen, n_words=n_words) #如果这个batch不满足，则跳过这个 if x is None: print 'Minibatch with zero sample under length ', maxlen uidx -= 1 continue ud_start = time.time() # compute cost, grads and copy grads to shared variables cost = f_grad_shared(x, x_mask) # do the update on parameters #更新参数，学习率0.01 f_update(lrate) ud = time.time() - ud_start # check for bad numbers if numpy.isnan(cost) or numpy.isinf(cost): print 'NaN detected' return 1. # verbose if numpy.mod(uidx, dispFreq) == 0: print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost, 'UD ', ud # save the best model so far if numpy.mod(uidx, saveFreq) == 0: print 'Saving...', if best_p is not None: params = best_p else: params = unzip(tparams) numpy.savez(saveto, history_errs=history_errs, **params) pkl.dump(model_options, open('%s.pkl' % saveto, 'wb')) print 'Done' # generate some samples with the model and display them if numpy.mod(uidx, sampleFreq) == 0: # FIXME: random selection? for jj in xrange(5): sample, score = gen_sample(tparams, f_next, model_options, trng=trng, maxlen=30, argmax=False) print 'Sample ', jj, ': ', ss = sample for vv in ss: if vv == 0: break if vv in worddicts_r: print worddicts_r[vv], else: print 'UNK', print # validate model on validation set and early stop if necessary #用valid数据进行交叉验证 if numpy.mod(uidx, validFreq) == 0: use_noise.set_value(0.) valid_errs = pred_probs(f_log_probs, prepare_data, model_options, valid) valid_err = valid_errs.mean() history_errs.append(valid_err) if uidx == 0 or valid_err <= numpy.array(history_errs).min(): best_p = unzip(tparams) bad_counter = 0 #如果120轮了，误差没有发生变化，则需要结束训练了 if len(history_errs) > patience and valid_err >= \ numpy.array(history_errs)[:-patience].min(): bad_counter += 1 if bad_counter > patience: print 'Early Stop!' estop = True break if numpy.isnan(valid_err): ipdb.set_trace() print 'Valid ', valid_err # finish after this many updates if uidx >= finish_after: print 'Finishing after %d iterations!' % uidx estop = True break #/* 每验证一次，记录下验证误差和测试误差， # 如果当前的验证误差大于前10(patience)次验证误差的最小值（也就是说误差没有降低），bad_counter+= 1， # 如果bad_counter>patience，就early stop！ print 'Seen %d samples' % n_samples if estop: break if best_p is not None: zipp(best_p, tparams) use_noise.set_value(0.) valid_err = pred_probs(f_log_probs, prepare_data, model_options, valid).mean() print 'Valid ', valid_err #保存最好的模型 params = copy.copy(best_p) numpy.savez(saveto, zipped_params=best_p, history_errs=history_errs, **params) return valid_err if __name__ == '__main__': pass
扫一扫
12-20
02-23
1749
1749
10-15
6389
6389
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
