在用BatchNorm层的时候有use_global_stats和moving_average_fraction不知道什么时候用,特别是在fine-tune的时候,因此把BatchNorm层的代码给解读一下。
————————————————————–caffe.proto——————————————————————–
首先来看BatchNorm有哪些参数。
message BatchNormParameter {
// If false, accumulate global mean/variance values via a moving average. If
// true, use those accumulated values instead of computing mean/variance
// across the batch.
optional bool use_global_stats = 1;
// How much does the moving average decay each iteration?
optional float moving_average_fraction = 2 [default = .999];
// Small value to add to the variance estimate so that we don't divide by
// zero.
optional float eps = 3 [default = 1e-5];
}
- use_global_stats :如果设置为true的话,就用累加的均值和方差而不重新计算;如果为false的话,则用moving average来计算均值与方差。
- moving_average_fraction:计算均值和方差时的比例,也就是动量。
- note:在训练的时候要将use_global_stats 设为false,测试的时候默认为true。
——————————————————-BatchNormLayer::LayerSetUp—————————————————
template <typename Dtype>
void BatchNormLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
//获取这一层的参数
BatchNormParameter param = this->layer_param_.batch_norm_param();
//获取moving_average_fraction
moving_average_fraction_ = param.moving_average_fraction();
//如果为测试,use_global_stats_ 为true,因此训练时为false
use_global_stats_ = this->phase_ == TEST;
if (param.has_use_global_stats())
use_global_stats_ = param.use_global_stats();
if (bottom[0]->num_axes() == 1)
channels_ = 1;
else
channels_ = bottom[0]->shape(1);//默认在channel上归一化
eps_ = param.eps();
//初始化参数
if (this->blobs_.size() > 0) {
LOG(INFO) << "Skipping parameter initialization";
} else {
this->blobs_.resize(3);
vector<int> sz;
sz.push_back(channels_);
//均值
this->blobs_[0].reset(new Blob<Dtype>(sz));
//方差
this->blobs_[1].reset(new Blob<Dtype>(sz));
sz[0] = 1;
//moving_average_fraction
this->blobs_[2].reset(new Blob<Dtype>(sz));
//初始化为0
for (int i = 0; i < 3; ++i) {
caffe_set(this->blobs_[i]->count(), Dtype(0),
this->blobs_[i]->mutable_cpu_data());
}
}
// Mask statistics from optimization by setting local learning rates
// for mean, variance, and the bias correction to zero.
//将学习率设为0,因为本层只是归一化操作,不需要更新参数
for (int i = 0; i < this->blobs_.size(); ++i) {
if (this->layer_param_.param_size() == i) {
ParamSpec* fixed_param_spec = this->layer_param_.add_param();
fixed_param_spec->set_lr_mult(0.f);
} else {
CHECK_EQ(this->layer_param_.param(i).lr_mult(), 0.f)
<< "Cannot configure batch normalization statistics as layer "
<< "parameters.";
}
}
}
——————————————————–BatchNormLayer::Reshape——————————————————
template <typename Dtype>
void BatchNormLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
if (bottom[0]->num_axes() >= 1)
CHECK_EQ(bottom[0]->shape(1), channels_);
//输出与输入大小一致
top[0]->ReshapeLike(*bottom[0]);
vector<int> sz;
sz.push_back(channels_);
//当前batch的均值与方差blob与channel个数一致
mean_.Reshape(sz);
variance_.Reshape(sz);
//一些中间变量
temp_.ReshapeLike(*bottom[0]);
x_norm_.ReshapeLike(*bottom[0]);
sz[0] = bottom[0]->shape(0);
batch_sum_multiplier_.Reshape(sz);
int spatial_dim = bottom[0]->count()/(channels_*bottom[0]->shape(0));
if (spatial_sum_multiplier_.num_axes() == 0 ||
spatial_sum_multiplier_.shape(0) != spatial_dim) {
sz[0] = spatial_dim;
spatial_sum_multiplier_.Reshape(sz);
Dtype* multiplier_data = spatial_sum_multiplier_.mutable_cpu_data();
caffe_set(spatial_sum_multiplier_.count(), Dtype(1), multiplier_data);
}
int numbychans = channels_*bottom[0]->shape(0);
if (num_by_chans_.num_axes() == 0 ||
num_by_chans_.shape(0) != numbychans) {
sz[0] = numbychans;
num_by_chans_.Reshape(sz);
caffe_set(batch_sum_multiplier_.count(), Dtype(1),
batch_sum_multiplier_.mutable_cpu_data());
}
}
——————————————————BatchNormLayer::Forward_cpu—————————————————
template <typename Dtype>
void BatchNormLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
const Dtype* bottom_data = bottom[0]->cpu_data();
Dtype* top_data = top[0]->mutable_cpu_data();
//batchsize
int num = bottom[0]->shape(0);
//HxW
int spatial_dim = bottom[0]->count()/(bottom[0]->shape(0)*channels_);
//先将输入拷贝到输出
if (bottom[0] != top[0]) {
caffe_copy(bottom[0]->count(), bottom_data, top_data);
}
//如果训练时use_global_stats_ = true,则不计算当前batch的均值和方差,用全局均值和方差代替(fine-tune)
if (use_global_stats_) {
// use the stored mean/variance estimates.
//首先计算scale = 1 / moving_average_fraction
const Dtype scale_factor = this->blobs_[2]->cpu_data()[0] == 0 ?
0 : 1 / this->blobs_[2]->cpu_data()[0];
//均值除以scale
caffe_cpu_scale(variance_.count(), scale_factor,
this->blobs_[0]->cpu_data(), mean_.mutable_cpu_data());
//方差除以scale
caffe_cpu_scale(variance_.count(), scale_factor,
this->blobs_[1]->cpu_data(), variance_.mutable_cpu_data());
}
//否则计算当前batch的均值和方差
else {
// compute mean
//先将HxW的feature map累加起来
caffe_cpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim,
1. / (num * spatial_dim), bottom_data,
spatial_sum_multiplier_.cpu_data(), 0.,
num_by_chans_.mutable_cpu_data());
//再将一个batch中按照channel维累加起来(计算的是channel上的均值)
caffe_cpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
num_by_chans_.cpu_data(), batch_sum_multiplier_.cpu_data(), 0.,
mean_.mutable_cpu_data());
}
// subtract mean
//将channel上的均值展开程numxchannel维
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
batch_sum_multiplier_.cpu_data(), mean_.cpu_data(), 0.,
num_by_chans_.mutable_cpu_data());
//再展开程numxchannelxHxW维, 并减去均值
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
spatial_dim, 1, -1, num_by_chans_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), 1., top_data);
//如果训练时use_global_stats_ = true,则不计算当前batch的均值和方差,用全局均值和方差代替(fine-tune
if (!use_global_stats_) {
// compute variance using var(X) = E((X-EX)^2)
//计算方差,先平方,在计算均值
caffe_powx(top[0]->count(), top_data, Dtype(2),
temp_.mutable_cpu_data()); // (X-EX)^2
//上面同样的方法计算平方的均值
caffe_cpu_gemv<Dtype>(CblasNoTrans, channels_ * num, spatial_dim,
1. / (num * spatial_dim), temp_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), 0.,
num_by_chans_.mutable_cpu_data());
caffe_cpu_gemv<Dtype>(CblasTrans, num, channels_, 1.,
num_by_chans_.cpu_data(), batch_sum_multiplier_.cpu_data(), 0.,
variance_.mutable_cpu_data()); // E((X_EX)^2)
// compute and save moving average
//计算moving_average_fraction,累加动量
this->blobs_[2]->mutable_cpu_data()[0] *= moving_average_fraction_;
this->blobs_[2]->mutable_cpu_data()[0] += 1;
//将当前batch的均值按照动量存入blob
caffe_cpu_axpby(mean_.count(), Dtype(1), mean_.cpu_data(),
moving_average_fraction_, this->blobs_[0]->mutable_cpu_data());
//将当前batch的方差按照动量存入blob
int m = bottom[0]->count()/channels_;
Dtype bias_correction_factor = m > 1 ? Dtype(m)/(m-1) : 1;
caffe_cpu_axpby(variance_.count(), bias_correction_factor,
variance_.cpu_data(), moving_average_fraction_,
this->blobs_[1]->mutable_cpu_data());
}
//计算均方差
// normalize variance
caffe_add_scalar(variance_.count(), eps_, variance_.mutable_cpu_data());
caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5),
variance_.mutable_cpu_data());
//按照上面同样的方法将方差展开成numxchannelxHxW维
// replicate variance to input size
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, num, channels_, 1, 1,
batch_sum_multiplier_.cpu_data(), variance_.cpu_data(), 0.,
num_by_chans_.mutable_cpu_data());
//temp中保存均方差
caffe_cpu_gemm<Dtype>(CblasNoTrans, CblasNoTrans, channels_ * num,
spatial_dim, 1, 1., num_by_chans_.cpu_data(),
spatial_sum_multiplier_.cpu_data(), 0., temp_.mutable_cpu_data());
//减去均值后的结果除以均方差
caffe_div(temp_.count(), top_data, temp_.cpu_data(), top_data);
// TODO(cdoersch): The caching is only needed because later in-place layers
// might clobber the data. Can we skip this if they won't?
//拷贝一份反向使用
caffe_copy(x_norm_.count(), top_data,
x_norm_.mutable_cpu_data());
}
—————————————————BatchNormLayer::Backward_cpu—————————————————–
反向推导公式请参考:https://www.cnblogs.com/LaplaceAkuir/p/7811351.html