Caffe生成的数据分为2种格式:Lmdb 和 Leveldb
- 它们都是键/值对(Key/Value Pair)嵌入式数据库管理系统编程库。
- 虽然lmdb的内存消耗是leveldb的1.1倍,但是lmdb的速度比leveldb快10%至15%,更重要的是lmdb允许多种训练模型同时读取同一组数据集。
- 因此lmdb取代了leveldb成为Caffe默认的数据集生成格式。
create_babyface.sh调用的convertData的源代码如下:
- #include<sys/types.h>
- #include<sys/stat.h>
- #include<dirent.h>
- #include <stdio.h>
- #include<string.h>
- #include <fstream> // NOLINT(readability/streams)
- #include <string>
- #include <vector>
- #include "boost/scoped_ptr.hpp"
- #include "glog/logging.h"
- #include "google/protobuf/text_format.h"
- #include "stdint.h"
- #include "caffe/proto/caffe.pb.h"
- #include "caffe/util/db.hpp"
- #include <opencv/cv.h>
- #include <opencv/highgui.h>
- using caffe::Datum;
- using boost::scoped_ptr;
- using std::string;
- namespace db = caffe::db;
- using namespace std;
- const int kCIFARSize = 32;
- const int kCIFARChannelBytes = 1024;
- const int kCIFARImageNBytes = 3072;
- const int kCIFARBatchSize = 1000;//1000 for a batch!
- const int kCIFARTrainBatches = 5;
- void read_image(std::ifstream* file, int* label, char* buffer) {
- char label_char;
- file->read(&label_char, 1);
- *label = label_char;
- file->read(buffer, kCIFARImageNBytes);
- return;
- }
- //Read IPLimage to the buffer
- void read_image(
- IplImage* out, char* buffer,
- char* RC, char* GC, char* BC)
- {
- int x,y;
- int idx =0;
- for(y = 0; y<out->height; y++){
- char *ptr= out->imageData + y * out->widthStep;
- for( x = 0;x< out->width;x++){
- idx =y*out->height + x;
- BC[idx]= ptr[3*x];
- GC[idx]= ptr[3*x+1];
- RC[idx]= ptr[3*x+2]; //这样就可以添加自己的操作,这里我使三通道颜色一样,就彩色图转黑白图了
- }
- }
- memcpy( buffer ,RC, kCIFARChannelBytes*sizeof(char) );
- memcpy( buffer+ kCIFARChannelBytes*sizeof(char) , GC,kCIFARChannelBytes*sizeof(char) );
- memcpy( buffer+ kCIFARChannelBytes*sizeof(char) *2, BC,kCIFARChannelBytes*sizeof(char) );
- return;
- }
- //Travel the folder and load the filelist!
- //使用linux dirent遍历目录
- int traveldir(char* path ,int depth, vector<string > &FileList)
- {
- DIR* d;// a
- struct dirent *file; struct stat sb;
- if( !(d=opendir(path ) ) ){
- printf("Read path %s error,wishchin! ", path);
- return -1;
- }
- while( (file= readdir(d ) ) != NULL ) {
- if(0== strncmp(file->d_name, ".", 1 ) ) continue;
- char filename[256];
- strcpy( filename , file->d_name );
- string Sfilename(filename);string Spath(path);
- Spath.append(Sfilename);
- FileList.push_back(Spath);
- }
- if( stat(file->d_name, &sb)>=0 && S_ISDIR(sb.st_mode) && depth <=4 )
- traveldir(file->d_name,depth+1,FileList);
- closedir(d);
- return 1;
- }
- // convert the data to the lmdb format !
- void convert_dataset(
- const string& input_folder,
- const string& output_folder,
- const string& db_type) {
- scoped_ptr<db::DB> train_db(db::GetDB(db_type));
- train_db->Open(output_folder + "/babyface_train_" + db_type, db::NEW);
- scoped_ptr<db::Transaction> txn(train_db->NewTransaction());
- char* path=new char[256];
- int depth=2;
- vector<string > FileList(0);
- // Data buffer
- int label;
- IplImage* ImageS;
- char str_buffer[kCIFARImageNBytes];
- char* RC=new char[kCIFARChannelBytes];
- char* GC=new char[kCIFARChannelBytes];
- char* BC=new char[kCIFARChannelBytes];
- Datum datum;
- datum.set_channels(3);
- datum.set_height(kCIFARSize);
- datum.set_width(kCIFARSize);
- //"Writing Training data"//载入训练数据
- LOG(INFO) << "Writing Training data";
- strcpy(path,( input_folder+(string)("train1") ).c_str() );
- traveldir( path , depth, FileList);
- for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
- // Open files
- LOG(INFO) << "Training Batch " << fileid + 1;
- snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
- //CHECK(data_file) << "Unable to open train file #" << fileid + 1;
- label=1;//The Batch has 10000 pics!
- for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
- ImageS =cvLoadImage( (FileList[ fileid*kCIFARTrainBatches + itemid] ).c_str() );
- read_image( ImageS, str_buffer, RC, GC, BC);
- datum.set_label(label);//datum.set_label(label);
- datum.set_data(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes,
- "%05d", fileid * kCIFARBatchSize + itemid);
- string out;
- CHECK(datum.SerializeToString( &out) ) ;
- txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!
- }
- }
- strcpy(path,( input_folder+(string)("train0") ).c_str() );
- traveldir( path , depth, FileList);
- for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
- LOG(INFO) << "Training Batch " << fileid + 1;
- snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
- //CHECK(data_file) << "Unable to open train file #" << fileid + 1;
- label=0;//The Batch has 10000 pics!
- for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
- ImageS =cvLoadImage( (FileList[ fileid*kCIFARTrainBatches + itemid] ).c_str() );
- read_image( ImageS, str_buffer, RC, GC, BC);
- datum.set_label(label);//datum.set_label(label);
- datum.set_data(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes,
- "%05d", fileid * kCIFARBatchSize + itemid);
- string out;
- CHECK(datum.SerializeToString( &out) ) ;
- txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!
- }
- }
- txn->Commit();
- train_db->Close();
- //写入测试数据!
- LOG(INFO) << "Writing Testing data";
- scoped_ptr<db::DB> test_db(db::GetDB(db_type));
- test_db->Open(output_folder + "/babyface_test_" + db_type, db::NEW);
- txn.reset(test_db->NewTransaction());
- strcpy(path,( input_folder+(string)("test1") ).c_str() );
- traveldir( path , depth, FileList);
- for (int fileid = 0; fileid < 2; ++fileid) {
- LOG(INFO) << "Training Batch " << fileid + 1;
- snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
- label=1;//The Batch has 10000 pics!
- for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
- ImageS =cvLoadImage( (FileList[ fileid*2 + itemid] ).c_str() );
- read_image( ImageS, str_buffer, RC, GC, BC);
- datum.set_label(label);//datum.set_label(label);
- datum.set_data(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes,
- "%05d", fileid * kCIFARBatchSize + itemid);
- string out;
- CHECK(datum.SerializeToString( &out) ) ;
- txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!
- }
- }
- strcpy(path,( input_folder+(string)("test0") ).c_str() );
- traveldir( path , depth, FileList);
- for (int fileid = 0; fileid < 2; ++fileid) {
- LOG(INFO) << "Training Batch " << fileid + 1;
- snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
- label=0;//The Batch has 10000 pics!
- for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
- ImageS =cvLoadImage( (FileList[ fileid*2 + itemid] ).c_str() );
- read_image( ImageS, str_buffer, RC, GC, BC);
- datum.set_label(label);//datum.set_label(label);
- datum.set_data(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes,
- "%05d", fileid * kCIFARBatchSize + itemid);
- string out;
- CHECK(datum.SerializeToString( &out) ) ;
- txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!
- }
- }
- txn->Commit();
- test_db->Close();
- cvReleaseImage(&ImageS);
- delete [] RC;delete [] GC;delete [] BC;
- }
- int main(int argc, char** argv) {
- if (argc != 4) {
- printf("This script converts the CIFAR dataset to the leveldb format used\n"
- "by caffe to perform classification.\n"
- "Usage:\n"
- " convert_cifar_data input_folder output_folder db_type\n"
- "Where the input folder should contain the binary batch files.\n"
- "The CIFAR dataset could be downloaded at\n"
- " http://www.cs.toronto.edu/~kriz/cifar.html\n"
- "You should gunzip them after downloading.\n");
- } else {
- google::InitGoogleLogging(argv[0]);
- convert_dataset(string(argv[1]), string(argv[2]), string(argv[3]));
- }
- return 0;
- }
后记:目的是载入32×32的三通道图像,直接输入3072维的char向量进行训练,至于怎样训练网络,还得仔细查看一下。
后记:代码出现 coredump 问题,利用 gcc path/...bin -o coredemo -g ,出现caffe.pb.h 包含丢失现象,why???