使用TensorFlow双流卷积神经网络对CK+表情数据库进行分类

本文所有代码已经放在GitHub上https://github.com/zhuzhuxia1994/CK-TensorFlow

双流卷积神经网络我最初是在做行为识别的时候接触到的，双流指的是时间流和空间流，再具体一点就是，时间流指的是对光流图片进行卷积处理，然后空间流指的是对RGB图片进行卷积处理，然后进行融合操作。这样往往比单纯对RGB图片进行卷积效果好，特别是在视频行为识别等方面，因为引入了时间信息。话不多说，放上代码及讲解。

以下代码是对表情识别CK+数据库进行实验，CK=数据库的地址如下http://www.consortium.ri.cmu.edu/ckagree/ 需要填写信息，如果有同学需要可以在博客下面留言，我会放上百度云地址~（先赞再跟我要哦，不然不理你，嘿嘿嘿）

 import tensorflow as tf

#%%
def inference(s_images,T_images, batch_size, n_classes):
    '''Build the model
    Args:
        images: image batch, 4D tensor, tf.float32, [batch_size, width, height, channels]
    Returns:
        output tensor with the computed logits, float, [batch_size, n_classes]
    '''
    #conv1, shape = [kernel size, kernel size, channels, kernel numbers]
    
    # one stream space
    with tf.variable_scope('s_conv1') as scope:
        weights = tf.get_variable('weights', 
                                  shape = [3,3,3, 16],
                                  dtype = tf.float32, 
                                  initializer=tf.truncated_normal_initializer(stddev=0.1,dtype=tf.float32))
        biases = tf.get_variable('biases', 
                                 shape=[16],
                                 dtype=tf.float32,
                                 initializer=tf.constant_initializer(0.1))
        conv = tf.nn.conv2d(s_images, weights, strides=[1,1,1,1], padding='SAME')
        pre_activation = tf.nn.bias_add(conv, biases)
        s_conv1 = tf.nn.relu(pre_activation, name= scope.name)
    
    #pool1 and norm1   
    with tf.variable_scope('s_pooling1_lrn') as scope:
        pool1 = tf.nn.max_pool(s_conv1, ksize=[1,3,3,1],strides=[1,2,2,1],
                               padding='SAME', name='s_pooling1')
        norm1 = tf.nn.lrn(pool1, depth_radius=4, bias=1.0, alpha=0.001/9.0,
                          beta=0.75,name='s_norm1')
    
    #conv2
    with tf.variable_scope('s_conv2') as scope:
        weights = tf.get_variable('weights',
                                  shape=[3,3,16,16],
                                  dtype=tf.float32,
                                  initializer=tf.truncated_normal_initializer(stddev=0.1,dtype=tf.float32))
        biases = tf.get_variable('biases',
                                 shape=[16], 
                                 dtype=tf.float32,
                                 initializer=tf.constant_initializer(0.1))
        conv = tf.nn.conv2d(norm1, weights, strides=[1,1,1,1],padding='SAME')
        pre_activation = tf.nn.bias_add(conv, biases)
        s_conv2 = tf.nn.relu(pre_activation, name='s_conv2')
    
    
    #pool2 and norm2
    with tf.variable_scope('s_pooling2_lrn') as scope:
        norm2 = tf.nn.lrn(s_conv2, depth_radius=4, bias=1.0, alpha=0.001/9.0,
                          beta=0.75,name='s_norm2')
        pool2 = tf.nn.max_pool(norm2, ksize=[1,3,3,1], strides=[1,1,1,1],
                               padding='SAME',name='s_pooling2')
    
    
    #local3
    with tf.variable_scope('s_local3') as scope:
        reshape = tf.reshape(pool2, shape=[batch_size, -1])
        dim = reshape.get_shape()[1].value
        weights = tf.get_variable('weights',
                                  shape=[dim,128],
                                  dtype=tf.float32,