之前设计的CNN有点问题,与传统的CNN有较大的差距。传统CNN不需要权重层的输出,下面就展示详细的实现。首先还是测试程序:
#include "cnn.hpp"
int main(int argc, char** argv)
{
using cnn_type = cnn <
nadam, sigmoid, XavierGaussian // 判别层使用的更新方法、激活函数和初始化方法
, 3, 64, 64, 4 // 3通道、64*64图像、4输出的判别层
, nadam, ReLu, XavierGaussian // 卷积池化层使用的更新方法、激活函数和初始化方法
, 5 // 5卷积核
, 8, 8 // 卷积核尺寸
, 4, 4 // 卷积核步幅
, 2, 2 // 池化层尺寸
, 5 // 5卷积核
, 4, 4 // 卷积核尺寸
, 2, 2 // 卷积核步幅
, 2, 2 // 池化层尺寸
, 5 // 3卷积核
, 2, 2 // 卷积核尺寸
, 1, 1 // 卷积核步幅
, 2, 2 // 池化层尺寸
>;
cnn_type cnn_layer;
cnn_type::input_type mti(.4);
cnn_type::ret_type mto(.8);
weight_initilizer<class def>::cal(mti, 0., 1.);
weight_initilizer<def>::cal(mto, 0, 1);
for (int i = 0; ; ++i)
{
auto mtok = cnn_layer.forward(mti);
cnn_layer.backward(mtok - mto);
if (i % 1000 == 0)
{
mto.print();
auto mmm = cnn_layer.forward(mti);
mmm.print();
_getch();
}
}
return 0;
}
结果如下,1000次训练以后能够完美匹配:
下面是其他改动,convolution_layer.hpp:
#ifndef _CONVOLUTION_LAYER_HPP_
#define _CONVOLUTION_LAYER_HPP_
#include "mat.hpp"
#include "weight_initilizer.hpp"
#include "base_function.hpp"
/* 卷积层 */
template<int input_row, int input_col, int tpl_row, int tpl_col, int row_step, int col_step, template<typename> class update_method_templ, template<typename> class activate_func, typename tpl_init_method, typename val_t = double>
struct conv_layer
{
using tpl_type = mat<tpl_row, tpl_col, val_t>;
using input_type = mat<input_row, input_col, val_t>;
using pad_type = mat<input_row + get_pad_size(input_row, tpl_row, row_step)
, input_col + get_pad_size(input_col, tpl_col, col_step), val_t>;
using pad_size = pad_size_t<input_row, input_col, tpl_row, tpl_col, row_step, col_step>;
using ret_type = decltype(inner_conv<row_step, col_step>(input_type().pad<pad_size::top, pad_size::left, pad_size::right, pad_size::bottom>(), tpl_type()));
tpl_type mt_tpl;
update_method_templ<mat<tpl_row, tpl_col, val_t>> um_tpl;
pad_type mt_input;
ret_type mt_bias;
update_method_templ<ret_type> um_bias;
activate_func<ret_type> act_func;
conv_layer()
{
weight_initilizer<tpl_init_method>::cal(mt_tpl);
}
inline ret_type forward(const input_type& mt)
{
mt_input = mt.pad<pad_size::top, pad_size::left, pad_size::right, pad_size::bottom>();
ret_type mt1 = inner_conv<row_step, col_step>(mt_input, mt_tpl);
return act_func.forward( mt1 + mt_bias);
}
inline input_type backward(const ret_type& mt_delta)
{
auto mt_delta_deact = act_func.backward() * mt_delta;
auto mt_delta_span = mt_delta_deact.span<row_step - 1, col_step - 1>(); // 采用了步长运算,等于有一些没计算,所以反向传播时候的贡献是0
using ret_pad_type = decltype(mt_delta_span);
/* 计算反向传播误差 */
/* 计算返回阵需要pad的大小 */
constexpr int target_r = tpl_row + pad_type::r - 1;
constexpr int target_c = tpl_col + pad_type::c - 1;
constexpr int pad_top = (target_r - ret_pad_type::r) / 2;
constexpr int pad_left = (target_c - ret_pad_type::c) / 2;
constexpr int pad_right = (target_c - ret_pad_type::c) - pad_left;
constexpr int pad_bottom = (target_r - ret_pad_type::r) - pad_top;
auto mt_delta_span_pad = mt_delta_span.pad<pad_top, pad_left, pad_right, pad_bottom>();
auto mt_tpl_rot = mt_tpl.rot180();
auto mt_ret_pad = inner_conv<1, 1, target_r, target_c, tpl_row, tpl_col, val_t>(mt_delta_span_pad, mt_tpl_rot);
input_type mt_ret;
mt_ret.assign<-1 * pad_size::top, -1 * pad_size::left>(mt_ret_pad); // 剪除外边
/* 计算卷积核更新 */
auto mt_update = inner_conv<1, 1, pad_type::r, pad_type::c, ret_pad_type::r, ret_pad_type::c, val_t>(mt_input, mt_delta_span);
if (mt_update.max_abs() != 0)
mt_update = mt_update / mt_update.max_abs();
mt_tpl = um_tpl.update(mt_tpl, mt_update);
mt_bias = mt_bias - um_bias.update(mt_delta_deact, mt_delta_deact);
/* 将模板均值置0,最大波动范围为1 */
double d_mean = mt_tpl.sum() / (tpl_row * tpl_col);
mt_tpl = mt_tpl - d_mean;
if (mt_tpl.max_abs() != 0)
mt_tpl = mt_tpl / mt_tpl.max_abs();
return mt_ret;
}
void print()
{
printf("<template>\r\n");
mt_tpl.print();
printf("<bias>\r\n");
mt_bias.print();
}
static void print_type()
{
printf("conv_layer<%d, %d, %d, %d, %d, %d> ", input_row, input_col, tpl_row, tpl_col, row_step, col_step);
input_type::print_type();
}
};
/*
多通道、多核的卷积层
输出是这个样子的:
| C1K1 C1K2 C1K3|
| C2K1 C2K2 C2K3|
| C3K1 C3K2 C3K3|
其中C是通道K是核,CK结果是一个卷积后的矩阵
*/
template<
int channel_num, int tpl_num
, int input_row, int input_col
, int tpl_row, int tpl_col
, int row_step, int col_step
, template<typename> class update_method_templ
, template<typename> class activate_func
, typename tpl_init_method
>
struct mul_channel_conv
{
using conv_type = conv_layer<input_row, input_col
, tpl_row, tpl_col
, row_step, col_step
, update_method_templ
, activate_func
, tpl_init_method
, double
>;
using input_type = mat<channel_num, 1, typename conv_type::input_type>;
using ret_type = mat<channel_num, tpl_num, typename conv_type::ret_type>;
conv_type tpls[tpl_num];
ret_type forward(const input_type& mt)
{
ret_type ret;
for (int i = 0; i < channel_num; ++i)
{
for (int j = 0; j < tpl_num; ++j)
{
ret.get(i, j) = tpls[j].forward(mt[i]);
}
}
return ret;
}
/* 这里返回比较特殊,因为返回的维度比较高,所以只能误差相加 */
input_type backward(const ret_type& delta)
{
input_type ret;
for (int i = 0; i < channel_num; ++i)
{
for (int j = 0; j < tpl_num; ++j)
{
ret.get(i, 0) = ret.get(i, 0) + tpls[j].backward(delta.get(i, j)) / (double)tpl_num;
}
}
return ret;
}
};
/* 多通道、多核的卷积层,最后将卷积结果进行加权,得到指定个数的结果 */
#include "bp.hpp"
template<
int channel_num, int tpl_num
, int input_row, int input_col
, int tpl_row, int tpl_col
, int row_step, int col_step
, template<typename> class update_method_templ
, template<typename> class activate_func
, typename tpl_init_method
>
struct mul_channel_conv_with_weight
{
using mul_conv_type = mul_channel_conv <
channel_num, tpl_num
, input_row, input_col
, tpl_row, tpl_col
, row_step, col_step
, update_method_templ
, activate_func
, tpl_init_method
>;
using mul_conv_ret_type = typename mul_conv_type::ret_type;
using weight_type = bp<typename mul_conv_ret_type::type, mul_conv_ret_type::c
, nadam, ReLu, HeGaussian
, mul_conv_ret_type::r, 1
>;
using ret_type = typename weight_type::ret_type::t_type;
using input_type = typename mul_conv_type::input_type;
mul_conv_type mconv;
weight_type weight;
ret_type forward(const input_type& mt)
{
return weight.forward(mconv.forward(mt)).t();
}
input_type backward(const ret_type& delta)
{
return mconv.backward(weight.backward(delta.t()));
}
};
#endif
卷积网络cnn.hpp
#ifndef _CNN_HPP_
#define _CNN_HPP_
#include "convolution_layer.hpp"
#include "pool_layer.hpp"
#include "bp.hpp"
/* 多核多通道加权卷积,再进行池化 */
template<
int channel_num, int tpl_num
, int input_row, int input_col
, int tpl_row, int tpl_col
, int row_step, int col_step
, int pool_row, int pool_col
, template<typename> class update_method_templ
, template<typename> class activate_func
, typename tpl_init_method
>
struct mccw_with_pool
{
using mccw_type = mul_channel_conv_with_weight<
channel_num, tpl_num
, input_row, input_col
, tpl_row, tpl_col
, row_step, col_step
, update_method_templ
, activate_func
, tpl_init_method
>;
using mccw_ret_type = typename mccw_type::ret_type::type;
using pool_type = pool_layer<pool_layer_max, mccw_ret_type::r, mccw_ret_type::c, pool_row, pool_col, typename mccw_ret_type::type>;
using ret_type = mat<tpl_num, 1, typename pool_type::ret_type>;
using input_type = typename mccw_type::input_type;
mccw_type mccw_layer;
pool_type pool;
ret_type forward(const input_type& mt)
{
ret_type ret;
for (int i = 0; i < tpl_num; ++i)
{
ret[i] = pool.forward(mccw_layer.forward(mt)[i]);
}
return ret;
}
input_type backward(const ret_type& delta)
{
using pool_out_type = typename mccw_type::ret_type;
pool_out_type pool_out;
for (int i = 0; i < tpl_num; ++i)
{
pool_out[i] = pool.backward(delta[i]);
}
return mccw_layer.backward(pool_out);
}
};
/* 多通道情况下得出的是各个矩阵,将各个矩阵最后做一个卷积,然后将卷积结果拉直送给判别层
int类型的参数必须是10*n
*/
template<int channel_num, int input_row, int input_col> // 能自动推倒的放在这里
struct mccwp_cluster_input
{
template<
template<typename> class update_method_templ
, template<typename> class activate_func
, typename tpl_init_method
, int tpl_num
, int tpl_row, int tpl_col
, int row_step, int col_step
, int pool_row, int pool_col
, int ...rest
>
struct mccwp_cluster
{
using cur_type = mccw_with_pool<
channel_num, tpl_num
, input_row, input_col
, tpl_row, tpl_col
, row_step, col_step
, pool_row, pool_col
, update_method_templ
, activate_func
, tpl_init_method
>;
using next_input_type = typename cur_type::ret_type::type;
/* 下层的通道数就是上层的输出 */
using next_type = typename mccwp_cluster_input<tpl_num, next_input_type::r, next_input_type::c>::template mccwp_cluster<
update_method_templ
, activate_func
, tpl_init_method
, rest...
>;
using input_type = typename cur_type::input_type;
using ret_type = typename next_type::ret_type;
cur_type cur_mccwp;
next_type next;
ret_type forward(const input_type& mt_input)
{
return next.forward(cur_mccwp.forward(mt_input));
}
input_type backward(const ret_type& delta)
{
return cur_mccwp.backward(next.backward(delta));
}
using stretch_type = unite_mat_t<ret_type>;
static stretch_type stretch(const ret_type& ret)
{
stretch_type mtd;
auto p = mtd.pval->p;
for (int i = 0; i < ret.r; ++i)
{
memcpy(p, ret[i].pval->p, sizeof(*p) * ret[i].r * ret[i].c);
p += (ret[i].r * ret[i].c);
}
return mtd;
}
static ret_type split(const stretch_type& st)
{
ret_type ret;
auto p = st.pval->p;
for (int i = 0; i < ret.r; ++i)
{
memcpy(ret[i].pval->p, p, sizeof(*p) * ret[i].r * ret[i].c);
p += (ret[i].r * ret[i].c);
}
return ret;
}
};
template<
template<typename> class update_method_templ
, template<typename> class activate_func
, typename tpl_init_method
, int tpl_num
, int tpl_row, int tpl_col
, int row_step, int col_step
, int pool_row, int pool_col
>
struct mccwp_cluster<
update_method_templ
, activate_func
, tpl_init_method
, tpl_num
, tpl_row, tpl_col
, row_step, col_step
, pool_row, pool_col
>
{
using cur_type = mccw_with_pool<
channel_num, tpl_num
, input_row, input_col
, tpl_row, tpl_col
, row_step, col_step
, pool_row, pool_col
, update_method_templ
, activate_func
, tpl_init_method
>;
using input_type = typename cur_type::input_type;
using ret_type = typename cur_type::ret_type;
cur_type cur_mccwp;
ret_type forward(const input_type& mt_input)
{
return cur_mccwp.forward(mt_input);
}
input_type backward(const ret_type& delta)
{
return cur_mccwp.backward(delta);
}
using stretch_type = unite_mat_t<ret_type>;
static stretch_type stretch(const ret_type& ret)
{
stretch_type mtd;
auto p = mtd.pval->p;
for (int i = 0; i < ret.r; ++i)
{
memcpy(p, ret[i].pval->p, sizeof(*p) * ret[i].r * ret[i].c);
p += (ret[i].r * ret[i].c);
}
return mtd;
}
};
};
template<
template<typename> class judge_update_method_templ
, template<typename> class judge_activate_func
, typename judge_tpl_init_method
, int channel_num, int input_row, int input_col, int output_num
, template<typename> class update_method_templ
, template<typename> class activate_func
, typename tpl_init_method
, int...rest
>
struct cnn
{
using mccwpc_type = typename mccwp_cluster_input<channel_num, input_row, input_col>::template mccwp_cluster<
update_method_templ, activate_func, tpl_init_method
, rest...
>;
using input_type = typename mccwpc_type::input_type;
using bp_input_type = typename mccwpc_type::stretch_type;
using bp_type = bp<double, 1, judge_update_method_templ, judge_activate_func, judge_tpl_init_method, bp_input_type::r, output_num>;
using ret_type = typename bp_type::ret_type;
mccwpc_type mccwpc_layer;
bp_type bp_layer;
ret_type forward(const input_type& mt)
{
auto mccwpc_out = mccwpc_layer.forward(mt);
auto bp_input = mccwpc_type::stretch(mccwpc_out);
return bp_layer.forward(bp_input);
}
auto backward(const ret_type& delta)
{
auto bp_back = bp_layer.backward(delta);
auto bp_back_split = mccwpc_type::split(bp_back);
return mccwpc_layer.backward(bp_back_split);
}
};
#endif