Tensorflow学习之旅（十）——降噪自编码

为什么叫去噪呢，是因为我们在之前输入的数据是加了高斯噪声的，但是我们在学习特征时是不能把噪声也学进去的，所以这里还实现了

隐层上用tf.nn.softplus()激活函数？不知道理解是否正确！

看代码吧!就是一个三层的自编码器，只不过在输入加上了噪声。

# -*- coding: utf-8 -*-
"""
Created on Tue Jun 20 12:59:16 2017

@author: ASUS

"""
#降噪自编码

import numpy as np
import sklearn.preprocessing as prep 
import tensorflow as tf 
from tensorflow.examples.tutorials.mnist import input_data

# 定义Xavier初始化函数  tf.random_uniform产生均匀分布
def xavier_init(fan_in, fan_out, constant = 1):
    low = -constant * np.sqrt(6.0 / (fan_in + fan_out))
    high = constant * np.sqrt(6.0 / (fan_in + fan_out))
    return tf.random_uniform((fan_in, fan_out), minval = low, maxval = high, dtype = tf.float32)
# 定义自编码器类 
class AdditiveGaussianNoiseAutoEncoder(object):
    def __init__(self, n_input, n_hidden, transfer_function = tf.nn.softplus,
        optimizer = tf.train.AdamOptimizer(), scale = 0.1):

        self.n_input = n_input
        self.n_hidden = n_hidden
        self.transfer = transfer_function
        self.scale = tf.placeholder(tf.float32)
        self.training_scale = scale 
        network_weights = self._initialize_weights()
        self.weights = network_weights
        # 定义网络结构 

        # x 为输入，因此要用placeholder进行“占位符”操作
        self.x = tf.placeholder(tf.float32, [None, self.n_input])

        # hidden 是隐含层 ，此自编码器只含一个隐层
        # self.x + scale * tf.random_normal((n_input,)) 是加噪声，scale是噪声系数
        #  hidden = f(w*x1+b)， f 是激活函数 
        #  weights w1和b1 分别表示第一层的权值、偏置
        self.hidden = self.transfer(tf.add(tf.matmul(
            self.x + scale * tf.random_normal((n_input,)),
            self.weights['w1']), self.weights['b1']))

        # 定义重构 （隐层到输出层）
        self.reconstruction = tf.add(tf.matmul(self.hidden,self.weights['w2']),
            self.weights['b2'])

        # 定义 平方误差为cost 
        # tf.pow()是计算幂 2.0则表示计算平方, tf.subtract是对应元素相减
        self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(
            self.reconstruction, self.x), 2.0))

        # 定义优化器 对损失 self.cost进行优化
        self.optimizer = optimizer.minimize(self.cost)
        # 全局参数初始化
        init = tf.global_variables_initializer()
        # 创建会话 sess
        self.sess = tf.Session()
        self.sess.run(init)

    # 定义权值初始化函数，AE的权值存放在一个字典里
    # w1采用xavier初始化，其余设置为全0 
    def _initialize_weights(self):
        all_weights = dict()
        all_weights['w1'] = tf.Variable(xavier_init(
            self.n_input, self.n_hidden))
        all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden],
            dtype = tf.float32))
        all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden,
            self.n_input], dtype = tf.float32))
        all_weights['b2'] = tf.Variable(tf.zeros([self.n_input],
            dtype = tf.float32))
        return all_weights 

    # 定义 执行一步训练的函数
    def partial_fit(self, X):
        cost, opt = self.sess.run((self.cost, self.optimizer),
            feed_dict = {self.x: X, self.scale: self.training_scale})
        return cost

    # 定义计算总的cost，在评测AE时用到
    # 只让Session执行一个计算图节点 self.cost
    def calc_total_cost(self, X):
        return self.sess.run(self.cost, feed_dict = {
            self.x : X, self.scale: self.training_scale
            })

    # 定义transform函数，
    # 作用是 返回AE隐含层的输出结果
    # 目的是 提供一个接口来获取抽象后的特征
    def transform(self, X):
        return self.sess.run(self.hidden, feed_dic = {
            self.scale: self.training_scale
            })

    # 定义generate函数，将隐含层的输出作为输入，
    # 通过重建层（reconstruction）来复原原始数据
    def generate(self, hidden = None):
        if hidden is None:
            hidden = np.random.normal(size = self.weights['b1'])
        return self.sess.run(self.reconstruction, 
            feed_dict = {self.hidden: hidden})

    # 定义重构函数
    # 包括抽象特征的提取和 通过抽象特征来复原原始数据
    def reconstruct(self, X):
        return self.sess.run(self.reconstruction, fedd_dict = {
            self.x: X, self.scale: self.traning_scale
            })

    # getWeights获取 隐含层权重
    def getWeights(self):
        return self.sess.run(self.weights['w1'])
    def getBiases(self):
        return self.sess.run(self.weights['b1'])


# 定义函数 对 mnist数据进行标准化 （减均值，除以标准差）
# 利用skleran里的 StandardScaler类 
def standard_scale(X_train, X_test):
    preprocessor = prep.StandardScaler().fit(X_train)
    X_train = preprocessor.transform(X_train)
    X_test = preprocessor.transform(X_test)
    return X_train, X_test

# 定义函数 获取随机block数据 
def get_random_block_from_data(data, batch_size):
    start_index = np.random.randint(0, len(data) - batch_size )
    return data[start_index:(start_index + batch_size)]

mnist = input_data.read_data_sets('MNIST_data', one_hot = True)
# 数据标准化 （减均值，除以标准差）
X_train, X_test = standard_scale(mnist.train.images, mnist.test.images)

# 设置基本参数
n_samples = int(mnist.train.num_examples)
training_epochs = 20
batch_size = 128
display_step = 1

# 创建AGN（Additive Gaussian Noise，加性高斯噪声）自编码器实例
autoencoder = AdditiveGaussianNoiseAutoEncoder(n_input = 784,
                    n_hidden = 200,
                    transfer_function = tf.nn.softplus,  #这个函数的作用是计算激活函数softplus,即log( exp( features ) + 1)。
                    optimizer = tf.train.AdamOptimizer(learning_rate = 0.001),
                    scale = 0.01)

# 迭代训练
for epoch in range(training_epochs): #20
    avg_cost = 0.
    total_batch = int(n_samples / batch_size)
    for i in range(total_batch):
        batch_xs = get_random_block_from_data(X_train, batch_size)
        cost = autoencoder.partial_fit(batch_xs)  #执行一步训练的函数
        avg_cost += cost / n_samples * batch_size
    if epoch % display_step ==0 :
        print('Epoch: ', '%04d' % (epoch+1), 'Cost = ','{:.9f}'.format(avg_cost))

print('Total cost: ' + str(autoencoder.calc_total_cost(X_test)))

下面是一个添加了在tensorboard显示计算图的代码以及效果
#自编码器，输入是784（28*28）维，中间解码为500维，最终在编码回784维
#保存一组输入，以及对应的输出，发现还原度很高
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data

import matplotlib # to plot images
# Force matplotlib to not use any X-server backend.
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec

## Visualizing reconstructions
def vis(images, save_name):
    dim = images.shape[0]
    n_image_rows = int(np.ceil(np.sqrt(dim)))
    n_image_cols = int(np.ceil(dim * 1.0/n_image_rows))
    gs = gridspec.GridSpec(n_image_rows,n_image_cols,top=1., bottom=0., right=1., left=0., hspace=0., wspace=0.)
    for g,count in zip(gs,range(int(dim))):
        ax = plt.subplot(g)
        ax.imshow(images[count,:].reshape((28,28)))
        ax.set_xticks([])
        ax.set_yticks([])
    plt.savefig(save_name + '_vis.png')

mnist_width = 28
n_visible = mnist_width * mnist_width
n_hidden = 500
corruption_level = 0.3

# create node for input data
X = tf.placeholder("float", [None, n_visible], name='X')

# create node for corruption mask  用于将部分输入数据置为0
mask = tf.placeholder("float", [None, n_visible], name='mask')

# create nodes for hidden variables
W_init_max = 4 * np.sqrt(6. / (n_visible + n_hidden))  #0.0156492  这是一种初始化方法，是最合适的随机分布
W_init = tf.random_uniform(shape=[n_visible, n_hidden],  #[784,500]
                           minval=-W_init_max,
                           maxval=W_init_max)

W = tf.Variable(W_init, name='W')
b = tf.Variable(tf.zeros([n_hidden]), name='b')

W_prime = tf.transpose(W)  # tied weights between encoder and decoder 只是初始的时候数据相同，训练起来就不同了
b_prime = tf.Variable(tf.zeros([n_visible]), name='b_prime')


def model(X, mask, W, b, W_prime, b_prime):
    tilde_X = mask * X  # corrupted X  这里是普通乘法，对应位相乘，不是矩阵乘法

    Y = tf.nn.sigmoid(tf.matmul(tilde_X, W) + b)  # hidden state
    Z = tf.nn.sigmoid(tf.matmul(Y, W_prime) + b_prime)  # reconstructed input
    return Z

# build model graph
Z = model(X, mask, W, b, W_prime, b_prime)

# create cost function
cost = tf.reduce_sum(tf.pow(X - Z, 2))  # minimize squared error
train_op = tf.train.GradientDescentOptimizer(0.02).minimize(cost)  # construct an optimizer
predict_op = Z
# load MNIST data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
#print(len(trX ))  #55000
#print(np.shape(trX))  #(55000,784)

# Launch the graph in a session
with tf.Session() as sess:
    # you need to initialize all variables
    tf.global_variables_initializer().run()

    for i in range(100):  #100
        for start, end in zip(range(0, len(trX), 128), range(128, len(trX)+1, 128)): #最后一组是54784 54912
            #print(start, end )
            input_ = trX[start:end]
            mask_np = np.random.binomial(1, 1 - corruption_level, input_.shape)  #同输入shape的一个二项分布，由0和1组成
            #print(mask_np)
            sess.run(train_op, feed_dict={X: input_, mask: mask_np})

        mask_np = np.random.binomial(1, 1 - corruption_level, teX.shape) 
        print(i, sess.run(cost, feed_dict={X: teX, mask: mask_np}))  #这里一次性将所有test数据都读入了 测试代价
    # save the predictions for 100 images
    mask_np = np.random.binomial(1, 1 - corruption_level, teX[:100].shape)
    predicted_imgs = sess.run(predict_op, feed_dict={X: teX[:100], mask: mask_np})
    input_imgs = teX[:100]
    # plot the reconstructed images
    vis(predicted_imgs,'pred')   #编码后的结果  不是中间层的输出
    vis(input_imgs,'in')          #输入
    print('Done')