先从最基础的Data层讲起。
看看datalayer相关的类的继承关系
首先定义了一个
template <typename Dtype>
class Batch {
public:
Blob<Dtype> data_, label_;
};
base_data_layer.cpp
#include <boost/thread.hpp>
#include <vector>
#include "caffe/blob.hpp"
#include "caffe/data_transformer.hpp"
#include "caffe/internal_thread.hpp"
#include "caffe/layer.hpp"
#include "caffe/layers/base_data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/blocking_queue.hpp"
namespace caffe {
//调用模板类layer的构造函数,将param传递给模板类layer的构造函数,同时用param.transform_param()初始化BaseDataLayer的transform_param_成员
template <typename Dtype>
BaseDataLayer<Dtype>::BaseDataLayer(const LayerParameter& param)
: Layer<Dtype>(param),
transform_param_(param.transform_param()) {
}
template <typename Dtype>
void BaseDataLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
if (top.size() == 1) {
output_labels_ = false;
} else {
output_labels_ = true;
}
data_transformer_.reset(
new DataTransformer<Dtype>(transform_param_, this->phase_));
data_transformer_->InitRand();
// The subclasses should setup the size of bottom and top
DataLayerSetUp(bottom, top);
}
template <typename Dtype>
BasePrefetchingDataLayer<Dtype>::BasePrefetchingDataLayer(
const LayerParameter& param)
: BaseDataLayer<Dtype>(param),
prefetch_free_(), prefetch_full_() {
for (int i = 0; i < PREFETCH_COUNT; ++i) {
prefetch_free_.push(&prefetch_[i]);//压入batch的地址
}
}
template <typename Dtype>
void BasePrefetchingDataLayer<Dtype>::LayerSetUp(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
//1. 调用父类BaseDataLayer构造方法,因为BasePrefetchingDataLayer没有覆盖BaseDataLayer的DataLayerSetUp方法,所有他仍然调用父类的DataLayerSetUp方法
BaseDataLayer<Dtype>::LayerSetUp(bottom, top);//回避虚函数的机制,通过作用域符来限定调用的虚函数版本。由于C++的多态性,BaseDataLayer<Dtype>::LayerSetUp(bottom, top)会调用Datalayer的ataLayerSetUp方法,该方法会Reshape prefetch_的batch的data_ blob、label_ blob
// Before starting the prefetch thread(预取线程), we make cpu_data and gpu_data
// calls so that the prefetch thread does not accidentally make simultaneous
// cudaMalloc calls when the main thread is running. In some GPUs this
// seems to cause failures if we do not so.
for (int i = 0; i < PREFETCH_COUNT; ++i) {
//2. 访问预取数据空间,这里是为了提前分配预取数据的存储空间
prefetch_[i].data_.mutable_cpu_data();//由于C++的多态性,BaseDataLayer<Dtype>::LayerSetUp(bottom, top)会调用Datalayer的ataLayerSetUp方法,该方法会Reshape prefetch_的batch的data_ blob、label_ blob
if (this->output_labels_) {
prefetch_[i].label_.mutable_cpu_data();
}
}
#ifndef CPU_ONLY
if (Caffe::mode() == Caffe::GPU) {
for (int i = 0; i < PREFETCH_COUNT; ++i) {
prefetch_[i].data_.mutable_gpu_data();
if (this->output_labels_) {
prefetch_[i].label_.mutable_gpu_data();
}
}
}
#endif
//3. 创建用于预取数据的线程
DLOG(INFO) << "Initializing prefetch";
this->data_transformer_->InitRand();//调用 DataTransformer类的InitRand方法生成一个随机数生成器。注意,DataTransformer类中有一个成员是shared_ptr<Caffe::RNG> rng_
StartInternalThread();//会调用InternalThreadEntry方法,因为Datalayer类没有覆盖InternalThreadEntry方法,所以如果真实的对象类型是Data_layer的话,也只会调用其基类的方法,即BasePrefetchingDataLayer<Dtype>::InternalThreadEntry()。
DLOG(INFO) << "Prefetch initialized.";
}
template <typename Dtype>
void BasePrefetchingDataLayer<Dtype>::InternalThreadEntry() {
#ifndef CPU_ONLY
cudaStream_t stream;
if (Caffe::mode() == Caffe::GPU) {
CUDA_CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
}
#endif
try {
while (!must_stop()) {
Batch<Dtype>* batch = prefetch_free_.pop();//batch是指针
//load_batch(Batch<Dtype>* batch)方法Reshape了其中的data_ Blob,为其重新分配所需的内存。做到这一点已经足够,因为prefetch_free_中存储的也只是指针
load_batch(batch);//实际上会调用DataLayer的load_batch方法,因为它是个纯虚函数
#ifndef CPU_ONLY
if (Caffe::mode() == Caffe::GPU) {
batch->data_.data().get()->async_gpu_push(stream);
CUDA_CHECK(cudaStreamSynchronize(stream));
}
#endif
prefetch_full_.push(batch);//batch在经过load_batch(batch)后发生了变化
}
} catch (boost::thread_interrupted&) {
// Interrupted exception is expected on shutdown
}
#ifndef CPU_ONLY
if (Caffe::mode() == Caffe::GPU) {
CUDA_CHECK(cudaStreamDestroy(stream));
}
#endif
}
//数据层作为网络的最底层,其forward功能只需要将设置top[0] top[1]的数据,即拷贝。
template <typename Dtype>
void BasePrefetchingDataLayer<Dtype>::Forward_cpu(
const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
Batch<Dtype>* batch = prefetch_full_.pop("Data layer prefetch queue empty");
// Reshape to loaded data.
top[0]->ReshapeLike(batch->data_);
// Copy the data
caffe_copy(batch->data_.count(), batch->data_.cpu_data(),
top[0]->mutable_cpu_data());//将数据从batch拷贝到top[0]
DLOG(INFO) << "Prefetch copied";
if (this->output_labels_) {
// Reshape to loaded labels.
top[1]->ReshapeLike(batch->label_);
// Copy the labels.
caffe_copy(batch->label_.count(), batch->label_.cpu_data(),
top[1]->mutable_cpu_data());//将数据从batch拷贝到top[1]
}
prefetch_free_.push(batch);
}
// 如果没有GPU的话则在BasePrefetchingDataLayer类中生成一个Forward函数
// 该函数并不前传,而是直接报错
#ifdef CPU_ONLY
STUB_GPU_FORWARD(BasePrefetchingDataLayer, Forward);
#endif
INSTANTIATE_CLASS(BaseDataLayer);
INSTANTIATE_CLASS(BasePrefetchingDataLayer);
//最后这两个再后面的几篇博客中有介绍到,可以看看定义<code class="prettyprint">INSTANTIATE_CLASS(</code>BaseDataLayer)被用来实例化BaseDataLayer的
<code class="prettyprint"></code>//类模板,<code class="prettyprint">REGISTER_LAYER_CLASS(</code>BaseData)被用来向layer_factory注册BaseDataLayer的构造方法,方便直接通过层的
<code class="prettyprint"></code>//名称(BaseData)直接获取层的对象。Caffe中内置的层在实现的码的最后都会加上这两个宏。
/*
<pre><code class="language-" data-lang="">// ------ in common.hpp ------
// Instantiate a class with float and double specifications.
#define INSTANTIATE_CLASS(classname) \
char gInstantiationGuard##classname; \
template class classname<float>; \
template class classname<double>
// ------ in common.hpp ------
// ------ in layer_factory.hpp ------
#define REGISTER_LAYER_CREATOR(type, creator) \
static LayerRegisterer<float> g_creator_f_##type(#type, creator<float>); \
static LayerRegisterer<double> g_creator_d_##type(#type, creator<double>) \
#define REGISTER_LAYER_CLASS(type) \
template <typename Dtype> \
shared_ptr<Layer<Dtype> > Creator_##type##Layer(const LayerParameter& param) \
{ \
return shared_ptr<Layer<Dtype> >(new type##Layer<Dtype>(param)); \
} \
REGISTER_LAYER_CREATOR(type, Creator_##type##Layer)
// ------ in layer_factory.hpp ------
} // namespace caffe
image_data_layer
#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>
#include <fstream> // NOLINT(readability/streams)
#include <iostream> // NOLINT(readability/streams)
#include <string>
#include <utility>
#include <vector>
#include "caffe/data_layers.hpp"
#include "caffe/layer.hpp"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/util/rng.hpp"
namespace caffe {
template <typename Dtype>
ImageDataLayer<Dtype>::~ImageDataLayer<Dtype>() {
this->StopInternalThread();
}
template <typename Dtype>
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
// 根据参数文件设置参数
// 图像的高度、宽度、是否彩色图像、图像目录
const int new_height = this->layer_param_.image_data_param().new_height();
const int new_width = this->layer_param_.image_data_param().new_width();
const bool is_color = this->layer_param_.image_data_param().is_color();
string root_folder = this->layer_param_.image_data_param().root_folder();
// 当前只支持读取高度和宽度同样大小的图像
CHECK((new_height == 0 && new_width == 0) ||
(new_height > 0 && new_width > 0)) << "Current implementation requires "
"new_height and new_width to be set at the same time.";
// Read the file with filenames and labels
// 读取存放图像文件名和类标的列表文件
const string& source = this->layer_param_.image_data_param().source();
LOG(INFO) << "Opening file " << source;
std::ifstream infile(source.c_str());
string filename;
int label;
// lines_存放文件名和类标的pair
while (infile >> filename >> label) {
lines_.push_back(std::make_pair(filename, label));
}
// 是否需要打乱文件的顺序
if (this->layer_param_.image_data_param().shuffle()) {
// randomly shuffle data
LOG(INFO) << "Shuffling data";
const unsigned int prefetch_rng_seed = caffe_rng_rand();
prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
ShuffleImages();
}
LOG(INFO) << "A total of " << lines_.size() << " images.";
// 随机跳过的图像,调过的图像个数在[0, rand_skip-1]之间
lines_id_ = 0;
// Check if we would need to randomly skip a few data points
// 如果参数中的rand_skip大于1,则随机跳过[0,rand_skip-1]个图片
//
if (this->layer_param_.image_data_param().rand_skip()) {
unsigned int skip = caffe_rng_rand() %
this->layer_param_.image_data_param().rand_skip();
LOG(INFO) << "Skipping first " << skip << " data points.";
CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
lines_id_ = skip;
}
// Read an image, and use it to initialize the top blob.
// 读取文件名到Mat
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_image.
// 对数据的形状进行推断
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
// 设置transformed_data_的形状
this->transformed_data_.Reshape(top_shape);
// Reshape prefetch_data and top[0] according to the batch_size.
// 设置batch_size
const int batch_size = this->layer_param_.image_data_param().batch_size();
CHECK_GT(batch_size, 0) << "Positive batch size required";
top_shape[0] = batch_size;
// 设置预取数组中数据的形状
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].data_.Reshape(top_shape);
}
// 设置输出的数据的形状
top[0]->Reshape(top_shape);
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
// 设置输出的类标的形状
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
// 设置预取数组中类标的形状
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].label_.Reshape(label_shape);
}
}
// 产生打乱图像顺序的数组
template <typename Dtype>
void ImageDataLayer<Dtype>::ShuffleImages() {
caffe::rng_t* prefetch_rng =
static_cast<caffe::rng_t*>(prefetch_rng_->generator());
shuffle(lines_.begin(), lines_.end(), prefetch_rng);
}
// This function is called on prefetch thread
// 该函数会被内部的线程调用
template <typename Dtype>
void ImageDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// 获取层参数,具体参见层参数的定义的解释
ImageDataParameter image_data_param = this->layer_param_.image_data_param();
const int batch_size = image_data_param.batch_size();
const int new_height = image_data_param.new_height();
const int new_width = image_data_param.new_width();
const bool is_color = image_data_param.is_color();
string root_folder = image_data_param.root_folder();
// Reshape according to the first image of each batch
// on single input batches allows for inputs of varying dimension.
// 读取跳过之后的第一幅图像,然后根据该图像设置相撞
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
// Use data_transformer to infer the expected blob shape from a cv_img.
// 推断图像形状
vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
// 设置transformed_data_形状
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
// 设置batch_size
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
Dtype* prefetch_data = batch->data_.mutable_cpu_data();
Dtype* prefetch_label = batch->label_.mutable_cpu_data();
// datum scales
// 读取一批图像,并进行预处理
const int lines_size = lines_.size();
for (int item_id = 0; item_id < batch_size; ++item_id) {
// get a blob
timer.Start();
CHECK_GT(lines_size, lines_id_);
cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
new_height, new_width, is_color);
CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
read_time += timer.MicroSeconds();
timer.Start();
// Apply transformations (mirror, crop...) to the image
// 进行预处理
// 根据图像的批次获得图像数据的偏移量
int offset = batch->data_.offset(item_id);
// 设置图像数据的指针到transformed_data_
this->transformed_data_.set_cpu_data(prefetch_data + offset);
// 进行预处理
this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
trans_time += timer.MicroSeconds();//统计预处理时间
// 复制类标到prefetch_label
prefetch_label[item_id] = lines_[lines_id_].second;
// go to the next iter
lines_id_++;
// 是否是图像目录中的最后一个图像
if (lines_id_ >= lines_size) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
lines_id_ = 0;
// 打乱图像索引的顺序
if (this->layer_param_.image_data_param().shuffle()) {
ShuffleImages();
}
}
}
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
// 预处理时间
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
INSTANTIATE_CLASS(ImageDataLayer);
REGISTER_LAYER_CLASS(ImageData);
} // namespace caffe
#endif // USE_OPENCV
data_layer
#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>
#endif // USE_OPENCV
#include <stdint.h>
#include <string>
#include <vector>
#include "caffe/common.hpp"
#include "caffe/data_layers.hpp"
#include "caffe/layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"
namespace caffe {
// 初始化DataReader,层参数
template <typename Dtype>
DataLayer<Dtype>::DataLayer(const LayerParameter& param)
: BasePrefetchingDataLayer<Dtype>(param),
reader_(param) {
}
// 析构函数停止内部线程
template <typename Dtype>
DataLayer<Dtype>::~DataLayer() {
this->StopInternalThread();
}
// 数据层的初始化
template <typename Dtype>
void DataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
const vector<Blob<Dtype>*>& top) {
// 从层参数中读取batch_size
const int batch_size = this->layer_param_.data_param().batch_size();
// Read a data point, and use it to initialize the top blob.
// 从reader_中获取一个数据
Datum& datum = *(reader_.full().peek());
// Use data_transformer to infer the expected blob shape from datum.
// 用数据来推断blob的形状存放到top_shape
vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
this->transformed_data_.Reshape(top_shape);
// Reshape top[0] and prefetch_data according to the batch_size.
// 既然获取了数据的形状(channel,height,width),那么这里再设置一下batch_size
// top_shape[0]=batch_size
// top_shape[1]=channel
// top_shape[2]=height
// top_shape[3]=width
top_shape[0] = batch_size;
// 根据形状设置top[0]的形状
top[0]->Reshape(top_shape);
// 设置预取数据的形状
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].data_.Reshape(top_shape);
}
LOG(INFO) << "output data size: " << top[0]->num() << ","
<< top[0]->channels() << "," << top[0]->height() << ","
<< top[0]->width();
// label
// 如果输出类标的话则把top[1]的形状也弄一下
if (this->output_labels_) {
vector<int> label_shape(1, batch_size);
top[1]->Reshape(label_shape);
for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
this->prefetch_[i].label_.Reshape(label_shape);
}
}
}
// This function is called on prefetch thread
// 这个函数是在自己定义的线程执行函数内部执行的
template<typename Dtype>
void DataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
CPUTimer batch_timer;
batch_timer.Start();
double read_time = 0;
double trans_time = 0;
CPUTimer timer;
CHECK(batch->data_.count());
CHECK(this->transformed_data_.count());
// Reshape according to the first datum of each batch
// on single input batches allows for inputs of varying dimension.
// 意思是像以下这种做法这样的话,每个batch的数据的维度可以不一样
// 从参数文件获取batch_size
const int batch_size = this->layer_param_.data_param().batch_size();
// 获取第一个数据
Datum& datum = *(reader_.full().peek());
// Use data_transformer to infer the expected blob shape from datum.
// 使用第一个数据推断blob的形状
vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
this->transformed_data_.Reshape(top_shape);
// Reshape batch according to the batch_size.
top_shape[0] = batch_size;
batch->data_.Reshape(top_shape);
// top_data存数据
Dtype* top_data = batch->data_.mutable_cpu_data();
Dtype* top_label = NULL; // suppress warnings about uninitialized variables
// top_label存类标
if (this->output_labels_) {
top_label = batch->label_.mutable_cpu_data();
}
// 对这批数据进行处理
for (int item_id = 0; item_id < batch_size; ++item_id) {
timer.Start();
// get a datum
Datum& datum = *(reader_.full().pop("Waiting for data"));
read_time += timer.MicroSeconds();
timer.Start();
// Apply data transformations (mirror, scale, crop...)
// 对于给定批的数据获取offset,这里调用的是给定batchid,然后获取offset
int offset = batch->data_.offset(item_id);
this->transformed_data_.set_cpu_data(top_data + offset);
this->data_transformer_->Transform(datum, &(this->transformed_data_));
// Copy label.
// 复制类标
if (this->output_labels_) {
top_label[item_id] = datum.label();
}
// 数据传输时间
trans_time += timer.MicroSeconds();
// 将数据指针压到free队列
reader_.free().push(const_cast<Datum*>(&datum));
}
timer.Stop();
batch_timer.Stop();
DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
DLOG(INFO) << " Read time: " << read_time / 1000 << " ms.";
DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}
INSTANTIATE_CLASS(DataLayer);
REGISTER_LAYER_CLASS(Data);
} // namespace caffe