深度学习 Python TensorFlow框架下实现RBM深度学习网络（学习笔记）

'''创建RBM基类，存放受限玻尔兹曼机'''
import tensorflow as tf
import numpy as np
import sys

def tf_xavier_init(fan_in,fan_out,*,const=1.0,dtype=np.float32):
    k=const*np.sqrt(6.0/(fan_in+fan_out))
    return tf.random_uniform((fan_in,fan_out),minval=-k,maxval=k,dtype=dtype)

class RBM:
    def __init__(self,
                 n_visible,#可见层的神经元数
                 n_hidden,#隐藏层的神经元数
                 learning_rate=0.01,
                 momentum=0.95,
                 xavier_const=1.0,
                 err_function='mse',#错误函数
                 use_tqdm=False,#为进度指示或者不使用tqdm软件包
                 tqdm=None):
        if not 0.0<=momentum<=1.0:
            raise ValueError('momentum should be in range [0, 1]')
        
        if err_function not in {'mse','cosine'}:
            raise ValueError('err_function should be either \'mse\' or \'cosine\'')
            
        self._use_tqdm=use_tqdm
        self._tqdm=None
        
        if use_tqdm or tqdm is not None:
            from tqdm import tqdm
            self._tqdm=tqdm
            
        self.n_visible = n_visible
        self.n_hidden = n_hidden
        self.learning_rate = learning_rate
        self.momentum = momentum
        
        #tf.placeholder()在模型中占位，分配必要的内存
        self.x=tf.placeholder(tf.float32,[None,self.n_visible])
        self.y=tf.placeholder(tf.float32,[None,self.n_hidden])
        
        #tf.Variable()变量初始化
        self.w=tf.Variable(tf_xavier_init(self.n_visible,self.n_hidden,const=xavier_const),dtype=tf.float32)
        self.visible_bias=tf.Variable(tf.zeros([self.n_visible]),dtype=tf.float32)
        self.hidden_bias=tf.Variable(tf.zeros([self.n_hidden]),dtype=tf.float32)
        
        self.delta_w=tf.Variable(tf.zeros([self.n_visible,self.n_hidden]),dtype=tf.float32)
        self.delta_visible_bias=tf.Variable(tf.zeros([self.n_visible]),dtype=tf.float32)
        self.delta_hidden_bias=tf.Variable(tf.zeros([self.n_hidden]),dtype=tf.float32)
        
        self.update_weights=None
        self.update_deltas=None
        self.compute_hidden=None
        self.compute_visible=None
        self.compute_visible_from_hidden=None
        
        self._initialize_vars()
        
        #assert 判断
        assert self.update_weights is not None
        assert self.update_deltas is not None
        assert self.compute_hidden is not None
        assert self.compute_visible is not None
        assert self.compute_visible_from_hidden is not None
        
        #tf.nn.l2_normalize()l2范化，reduce_mean()，reduce_sum()归约计算
        if err_function=='cosine':
            x1_norm=tf.nn.l2_normalize(self.x,1)
            x2_norm=tf.nn.l2_normalize(self.compute_visible,1)
            cos_val=tf.reduce_mean(tf.reduce_sum(tf.mul(x1_norm,x2_norm),1))
            self.compute_err=tf.acos(cos_val)/tf.constant(np.pi)
        else:
            self.compute_err=tf.reduce_mean(tf.square(self.x-self.compute_visible))
        
        #将所有图变量进行集体初始化
        init=tf.global_variables_initializer()
        self.sess=tf.Session()
        self.sess.run(init)
        
    def _initialize_vars(self):
        pass
    
    #feed_dict()向占位符喂入数据
    def get_err(self,batch_x):
        return self.sess.run(self.compute_err,feed_dict={self.x:batch_x})
    
    def reconstruct(self,batch_x):
        return self.sess.run(self.compute_visible,feed_dict={self.x:batch_x})
    
    def partial_fit(self,batch_x):
        self.sess.run(self.update_weights+self.update_deltas,feed_dict={self.x:batch_x})

    def fit(self,
            data_x,#形状数据
            n_epoches=10,#迭代次数
            batch_size=10,#批量大小
            shuffle=True,#shuffle数据
            verbose=True#输出到标准输出
           ):
        assert n_epoches>0
        
        n_data=data_x.shape[0]
        
        if batch_size>0:
            n_batches=n_data//batch_size+(0 if n_data%batch_size==0 else 1)
        else:
            n_batches=1
            
        if shuffle:
            data_x_cpy=data_x.copy()
            inds=np.arange(n_data)
        else:
            data_x_cpy=data_x
            
        errs=[]
        
        for e in range(n_epoches):
            if verbose and not self._use_tqdm:
                print('Epoch:{:d}'.format(e))
            
            epoch_errs=np.zeros((n_batches,))
            epoch_errs_ptr=0
            
            #random.shuffle()所有元素随机排序
            if shuffle:
                np.random.shuffle(inds)
                data_x_cpy=data_x_cpy[inds]
                
            r_batches=range(n_batches)
            
            if verbose and self._use_tqdm:
                r_batches=self._tqdm(r_batches,desc='Epoch:{:d}'.format(e),ascii=True,file=sys.stdout)
                
            for b in r_batches:
                batch_x=data_x_cpy[b*batch_size:(b+1)*batch_size]
                self.partial_fit(batch_x)
                batch_err=self.get_err(batch_x)
                epoch_errs[epoch_errs_ptr]=batch_err
                epoch_errs_ptr+=1
                
            if verbose:
                err_mean=epoch_errs.mean()
                if self._use_tqdm:
                    self._tqdm.write('Train error:{:.4f}'.format(err_mean))
                    self._tqdm.write('')
                else:
                    print('Train error:{:.4f}'.format(err_mean))
                    print('')
                sys.stdout.flush()
                
            errs=np.hstack([errs,epoch_errs])
            
        return errs
        
'''在RBM基础上创建一个伯努利RBM深度网络结构，用类保存结构，该类继承于RBM类，根据不同的受限玻尔兹曼机的模型结构进行不一样的参数初始化'''    
#tf.nn.relu()激活函数，大于0的保持不变，小于0的数置为0
def sample_bernoulli(probs):
    return tf.nn.relu(tf.sign(probs-tf.random_uniform(tf.shape(probs))))

class BBRBM(RBM):
    def __init__(self,*args,**kwargs):
        RBM.__init__(self,*args,**kwargs)
        
    #tf.nn.sigmoid(x) y=1/(1+exp(-x))
    def _initialize_vars(self):
        hidden_p=tf.nn.sigmoid(tf.matmul(self.x,self.w)+self.hidden_bias)
        visible_recon_p=tf.nn.sigmoid(tf.matmul(sample_bernoulli(hidden_p),tf.transpose(self.w))+self.visible_bias)
        hidden_recon_p=tf.nn.sigmoid(tf.matmul(visible_recon_p,self.w)+self.hidden_bias)
        
        positive_grad=tf.matmul(tf.transpose(self.x),hidden_p)
        negative_grad=tf.matmul(tf.transpose(visible_recon_p),hidden_recon_p)
        
        def f(x_old,x_new):
            return self.momentum*x_old+self.learning_rate * x_new * (1 - self.momentum) / tf.to_float(tf.shape(x_new)[0])
        
        delta_w_new = f(self.delta_w, positive_grad - negative_grad)
        delta_visible_bias_new = f(self.delta_visible_bias, tf.reduce_mean(self.x - visible_recon_p, 0))
        delta_hidden_bias_new = f(self.delta_hidden_bias, tf.reduce_mean(hidden_p - hidden_recon_p, 0))
        
        #assign()对变量进行更新
        update_delta_w = self.delta_w.assign(delta_w_new)
        update_delta_visible_bias = self.delta_visible_bias.assign(delta_visible_bias_new)
        update_delta_hidden_bias = self.delta_hidden_bias.assign(delta_hidden_bias_new)

        update_w = self.w.assign(self.w + delta_w_new)
        update_visible_bias = self.visible_bias.assign(self.visible_bias + delta_visible_bias_new)
        update_hidden_bias = self.hidden_bias.assign(self.hidden_bias + delta_hidden_bias_new)

        self.update_deltas = [update_delta_w, update_delta_visible_bias, update_delta_hidden_bias]
        self.update_weights = [update_w, update_visible_bias, update_hidden_bias]

        self.compute_hidden = tf.nn.sigmoid(tf.matmul(self.x, self.w) + self.hidden_bias)
        self.compute_visible = tf.nn.sigmoid(tf.matmul(self.compute_hidden, tf.transpose(self.w)) + self.visible_bias)
        self.compute_visible_from_hidden = tf.nn.sigmoid(tf.matmul(self.y, tf.transpose(self.w)) + self.visible_bias)

import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data

'''创建主模块，对RBM网络结构进行训练和测试'''
#导入minist手写数据集
mnist=input_data.read_data_sets('/mnist',one_hot=True)
mnist_images=mnist.train.images

#初始化RBM模型
model=BBRBM(n_visible=784,n_hidden=64,learning_rate=0.01,momentum=0.95,use_tqdm=True)
#训练模型，返回误差,n_epoches迭代次数,batch_size批量大小,尽可能小
errs=model.fit(mnist_images,n_epoches=30,batch_size=10)

#绘制模型的误差
plt.plot(errs)
plt.show()

#定义原始数据的显示格式，并指定要使用的原始数据
def show_digit(x):
    plt.imshow(x.reshape((28,28)),cmap=plt.cm.gray)
    plt.show()
    
IMAGE=1    
image=mnist_images[IMAGE]
show_digit(image)

#使用RBM模型的图像重构函数，生成训练结果图并输出
image_rec=model.reconstruct(image.reshape(1,-1))
show_digit(image_rec)

误差图：

原图：

生成的训练结果图：

IMAGE=2   
image=mnist_images[IMAGE]
show_digit(image)

#使用RBM模型的图像重构函数，生成训练结果图并输出
image_rec=model.reconstruct(image.reshape(1,-1))
show_digit(image_rec)

原图：

生成的训练结果图：