q1_window.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Q1: A window into NER
"""
from __future__ import absolute_import
from __future__ import division
import argparse
import sys
import time
import logging
from datetime import datetime
import tensorflow as tf
from util import print_sentence, write_conll
from data_util import load_and_preprocess_data, load_embeddings, read_conll, ModelHelper
from ner_model import NERModel
from defs import LBLS
#from report import Report
logger = logging.getLogger("hw3.q1")
logger.setLevel(logging.DEBUG)
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
class Config:
"""Holds model hyperparams and data information.
The config class is used to store various hyperparameters and dataset
information parameters. Model objects are passed a Config() object at
instantiation.
TODO: Fill in what n_window_features should be, using n_word_features and window_size.
"""
n_word_features = 2 # Number of features for every word in the input.
window_size = 1 # The size of the window to use.
### YOUR CODE HERE
n_window_features = n_word_features*(2*window_size+1)# The total number of features used for each window.
### END YOUR CODE
n_classes = 5
dropout = 0.5
embed_size = 50
hidden_size = 200
batch_size = 2048
n_epochs = 10
lr = 0.001
def __init__(self, output_path=None):
if output_path:
# Where to save things.
self.output_path = output_path
else:
self.output_path = "results/window/{:%Y%m%d_%H%M%S}/".format(datetime.now())
self.model_output = self.output_path + "model.weights"
self.eval_output = self.output_path + "results.txt"
self.log_output = self.output_path + "log"
self.conll_output = self.output_path + "window_predictions.conll"
def make_windowed_data(data, start, end, window_size = 1):
"""Uses the input sequences in @data to construct new windowed data points.
TODO: In the code below, construct a window from each word in the
input sentence by concatenating the words @window_size to the left
and @window_size to the right to the word. Finally, add this new
window data point and its label. to windowed_data.
Args:
data: is a list of (sentence, labels) tuples. @sentence is a list
containing the words in the sentence and @label is a list of
output labels. Each word is itself a list of
@n_features features. For example, the sentence "Chris
Manning is amazing" and labels "PER PER O O" would become
([[1,9], [2,9], [3,8], [4,8]], [1, 1, 4, 4]). Here "Chris"
the word has been featurized as "[1, 9]", and "[1, 1, 4, 4]"
is the list of labels.
start: the featurized `start' token to be used for windows at the very
beginning of the sentence.
end: the featurized `end' token to be used for windows at the very
end of the sentence.
window_size: the length of the window to construct.
Returns:
a new list of data points, corresponding to each window in the
sentence. Each data point consists of a list of
@n_window_features features (corresponding to words from the
window) to be used in the sentence and its NER label.
If start=[5,8] and end=[6,8], the above example should return
the list
[([5, 8, 1, 9, 2, 9], 1),
([1, 9, 2, 9, 3, 8], 1),
...
]
"""
windowed_data = []
for sentence, labels in data:
### YOUR CODE HERE (5-20 lines)
for i in range(len(sentence)):
x=[]
for j in range(2*window_size+1):
t = i - window_size + j
if t>=0 and t<len(sentence):
x.extend(sentence[t])
elif t<0:
x.extend(start)
else:
x.extend(end)
windowed_data.append((x,labels[i]))
### END YOUR CODE
return windowed_data
class WindowModel(NERModel):
"""
Implements a feedforward neural network with an embedding layer and
single hidden layer.
This network will predict what label (e.g. PER) should be given to a
given token (e.g. Manning) by using a featurized window around the token.
"""
def add_placeholders(self):
"""Generates placeholder variables to represent the input tensors
These placeholders are used as inputs by the rest of the model building and will be fed
data during training. Note that when "None" is in a placeholder's shape, it's flexible
(so we can use different batch sizes without rebuilding the model).
Adds following nodes to the computational graph
input_placeholder: Input placeholder tensor of shape (None, n_window_features), type tf.int32
labels_placeholder: Labels placeholder tensor of shape (None,), type tf.int32
dropout_placeholder: Dropout value placeholder (scalar), type tf.float32
Add these placeholders to self as the instance variables
self.input_placeholder
self.labels_placeholder
self.dropout_placeholder
(Don't change the variable names)
"""
### YOUR CODE HERE (~3-5 lines)
self.input_placeholder=tf.placeholder(tf.int32,shape=(None,self.config.n_window_features))
self.labels_placeholder=tf.placeholder(tf.int32,shape=(None,))
self.dropout_placeholder=tf.placeholder(tf.float32)
### END YOUR CODE
def create_feed_dict(self, inputs_batch, labels_batch=None, dropout=1):
"""Creates the feed_dict for the model.
A feed_dict takes the form of:
feed_dict = {
<placeholder>: <tensor of values to be passed for placeholder>,
....
}
Hint: The keys for the feed_dict should be a subset of the placeholder
tensors created in add_placeholders.
Hint: When an argument is None, don't add it to the feed_dict.
Args:
inputs_batch: A batch of input data.
labels_batch: A batch of label data.
dropout: The dropout rate.
Returns:
feed_dict: The feed dictionary mapping from placeholders to values.
"""
### YOUR CODE HERE (~5-10 lines)
feed_dict=dict()
if inputs_batch is not None:
feed_dict[self.input_placeholder]=inputs_batch
if labels_batch is not None:
feed_dict[self.labels_placeholder]=labels_batch
if dropout is not None:
feed_dict[self.dropout_placeholder]=dropout
### END YOUR CODE
return feed_dict
def add_embedding(self):
"""Adds an embedding layer that maps from input tokens (integers) to vectors and then
concatenates those vectors:
- Creates an embedding tensor and initializes it with self.pretrained_embeddings.
- Uses the input_placeholder to index into the embeddings tensor, resulting in a
tensor of shape (None, n_window_features, embedding_size).
- Concatenates the embeddings by reshaping the embeddings tensor to shape
(None, n_window_features * embedding_size).
Hint: You might find tf.nn.embedding_lookup useful.
Hint: You can use tf.reshape to concatenate the vectors. See following link to understand
what -1 in a shape means.
https://www.tensorflow.org/api_docs/python/array_ops/shapes_and_shaping#reshape.
Returns:
embeddings: tf.Tensor of shape (None, n_window_features*embed_size)
"""
### YOUR CODE HERE (!3-5 lines)
emb=tf.Variable(self.pretrained_embeddings)
embedding=tf.nn.embedding_lookup(emb, self.input_placeholder)
embeddings=tf.reshape(embedding,(-1,self.config.n_window_features*self.config.embed_size))
### END YOUR CODE
return embeddings
def add_prediction_op(self):
"""Adds the 1-hidden-layer NN:
h = Relu(xW + b1)
h_drop = Dropout(h, dropout_rate)
pred = h_dropU + b2
Recall that we are not applying a softmax to pred. The softmax will instead be done in
the add_loss_op function, which improves efficiency because we can use
tf.nn.softmax_cross_entropy_with_logits
When creating a new variable, use the tf.get_variable function
because it lets us specify an initializer.
Use tf.contrib.layers.xavier_initializer to initialize matrices.
This is TensorFlow's implementation of the Xavier initialization
trick we used in last assignment.
Note: tf.nn.dropout takes the keep probability (1 - p_drop) as an argument.
The keep probability should be set to the value of dropout_rate.
Returns:
pred: tf.Tensor of shape (batch_size, n_classes)
"""
x = self.add_embedding()
dropout_rate = self.dropout_placeholder
### YOUR CODE HERE (~10-20 lines)
xinit=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32)
W=tf.get_variable(name="W",dtype=tf.float32,shape=(self.config.n_window_features*self.config.embed_size,self.config.hidden_size),initializer=xinit)
b1=tf.get_variable(name="b1",dtype=tf.float32,shape=(self.config.hidden_size), initializer=xinit)
U=tf.get_variable(name="U",dtype=tf.float32,shape=(self.config.hidden_size,self.config.n_classes),initializer=xinit)
b2=tf.get_variable(name="b2",dtype=tf.float32,shape=(self.config.n_classes), initializer=xinit)
h=tf.nn.relu(tf.add(tf.matmul(x,W),b1))
h_drop=tf.nn.dropout(h, dropout_rate)
pred=tf.add(tf.matmul(h_drop,U),b2)
### END YOUR CODE
return pred
def add_loss_op(self, pred):
"""Adds Ops for the loss function to the computational graph.
In this case we are using cross entropy loss.
The loss should be averaged over all examples in the current minibatch.
Remember that you can use tf.nn.sparse_softmax_cross_entropy_with_logits to simplify your
implementation. You might find tf.reduce_mean useful.
Args:
pred: A tensor of shape (batch_size, n_classes) containing the output of the neural
network before the softmax layer.
Returns:
loss: A 0-d tensor (scalar)
"""
### YOUR CODE HERE (~2-5 lines)
probs=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred,labels=self.labels_placeholder)
loss=tf.reduce_mean(probs,axis=None)
### END YOUR CODE
return loss
def add_training_op(self, loss):
"""Sets up the training Ops.
Creates an optimizer and applies the gradients to all trainable variables.
The Op returned by this function is what must be passed to the
`sess.run()` call to cause the model to train. See
https://www.tensorflow.org/versions/r0.7/api_docs/python/train.html#Optimizer
for more information.
Use tf.train.AdamOptimizer for this model.
Calling optimizer.minimize() will return a train_op object.
Args:
loss: Loss tensor, from cross_entropy_loss.
Returns:
train_op: The Op for training.
"""
### YOUR CODE HERE (~1-2 lines)
optimizer = tf.train.AdamOptimizer(self.config.lr)
train_op = optimizer.minimize(loss)
### END YOUR CODE
return train_op
以上代码可用$ python q1_window.py test1和$python q1_window.py test2来debug。
q2_rnn_cell.py
class RNNCell(tf.nn.rnn_cell.RNNCell):
"""Wrapper around our RNN cell implementation that allows us to play
nicely with TensorFlow.
"""
def __init__(self, input_size, state_size):
self.input_size = input_size
self._state_size = state_size
@property
def state_size(self):
return self._state_size
@property
def output_size(self):
return self._state_size
def __call__(self, inputs, state, scope=None):
"""Updates the state using the previous @state and @inputs.
Remember the RNN equations are:
h_t = sigmoid(x_t W_x + h_{t-1} W_h + b)
TODO: In the code below, implement an RNN cell using @inputs
(x_t above) and the state (h_{t-1} above).
- Define W_x, W_h, b to be variables of the apporiate shape
using the `tf.get_variable' functions. Make sure you use
the names "W_x", "W_h" and "b"!
- Compute @new_state (h_t) defined above
Tips:
- Remember to initialize your matrices using the xavier
initialization as before.
Args:
inputs: is the input vector of size [None, self.input_size]
state: is the previous state vector of size [None, self.state_size]
scope: is the name of the scope to be used when defining the variables inside.
Returns:
a pair of the output vector and the new state vector.
"""
scope = scope or type(self).__name__
# It's always a good idea to scope variables in functions lest they
# be defined elsewhere!
with tf.variable_scope(scope):
### YOUR CODE HERE (~6-10 lines)
xinit=tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32)
W_x=tf.get_variable(name="W_x",dtype=tf.float32,shape=(self.input_size,self.state_size),initializer=xinit)
W_h=tf.get_variable(name="W_h",dtype=tf.float32,shape=(self.state_size,self.state_size),initializer=xinit)
b=tf.get_variable(name="b",dtype=tf.float32,shape=(self.state_size),initializer=xinit)
mul1=tf.matmul(inputs,W_x)
mul2=tf.matmul(state,W_h)
x_add_h=tf.add(mul1,mul2)
new_state=tf.nn.sigmoid(tf.add(x_add_h,b))
### END YOUR CODE ###
# For an RNN , the output and state are the same (N.B. this
# isn't true for an LSTM, though we aren't using one of those in
# our assignment)
output = new_state
return output, new_state以上代码可用$ python q2_rnn_cell.py test来debug。
扫一扫
1986
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
