索引处理类 - remove 方法实现
#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_
#include "common.h"
#include "mmap_file_op.h"
namespace qiniu
{
namespace largefile
{
struct IndexHeader
{
public:
IndexHeader()
{
memset(this, 0, sizeof(IndexHeader));
}
Blockinfo block_info_; /* meta block info */
int32_t bucket_size_; /* hash bucket size */
int32_t data_file_offset_; /* offset to write next data in block */
int32_t index_file_size_; /* offset after index_header + all bucketa */
int32_t free_head_offset_; /* free meta node list, for reuse */
};
class IndexHandle
{
public:
IndexHandle(const std::string &base_path, const uint32_t main_block_id);
~IndexHandle();
int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
int load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
/* remove index unmmap and file */
int remove(const uint32_t logic_block_id);
int flush();
IndexHeader *index_header()
{
return reinterpret_cast<IndexHeader *>(file_op_->get_map_data());
}
Blockinfo *block_info()
{
return reinterpret_cast<Blockinfo *>(file_op_->get_map_data());
}
int32_t bucket_size() const
{
//return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->bucket_size;
reinterpret_cast<IndexHeader *>(file_op_->get_map_data())->bucket_size_;
}
private:
MMapFileOperation *file_op_;
bool is_load_;
};
}
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */
#include "common.h"
#include "index_handle.h"
#include "sstream"
static int debug = 1;
namespace qiniu
{
namespace largefile
{
/* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
{
/* create file_op_ handle object */
std::stringstream tmp_stream;
tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */
std::string index_path;
tmp_stream >> index_path;
file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
is_load_ = false;
}
IndexHandle::~IndexHandle()
{
if (file_op_)
{
delete file_op_;
file_op_ = NULL;
}
}
int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
{
int ret = TFS_SUCCESS;
if (debug)
{
printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
}
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return TFS_ERROR;
}else if(file_size == 0) /* empty file */
{
IndexHeader i_header;
i_header.block_info_.block_id_ = logic_block_id;
i_header.block_info_.seq_no_ = 1;
i_header.bucket_size_ = bucket_size;
i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);
/* index header + total buckets */
char *init_data = new char[i_header.index_file_size_];
memcpy(init_data, &i_header, sizeof(IndexHandle));
memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));
/* write index header and buckets into file */
ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);
delete []init_data;
init_data = NULL;
if (ret != TFS_ERROR)
{
return ret;
}
ret = file_op_->flush_file();
if (ret != TFS_SUCCESS)
{
return ret;
}
}else /* file size > 0, index already exist */
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}
ret = file_op_->mmap_file(map_optiont);
if (ret != TFS_SUCCESS)
{
return ret;
}
is_load_ = true;
if (debug)
{
printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
{
int ret = TFS_SUCCESS;
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return file_size;
}
else if (file_size == 0) /* empty file */
{
return EXIT_INDEX_CORRUPT_ERROR;
}
MMapOption tmp_map_option = map_option;
if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
{
tmp_map_option.first_mmap_size_ = file_size;
}
ret = file_op_->mmap_file(tmp_map_option);
if (ret != TFS_SUCCESS)
{
return ret;
}
if (0 == bucket_size() || 0 == block_info()->block_id_)
{
fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check file size */
int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);
if (file_size < index_file_size)
{
fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check block_id_ size */
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
/* check bucket_size */
if (_bucket_size != bucket_size())
{
fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
return EXIT_BUCKET_CONFIGURE_ERROR;
}
is_load_ = true;
if (debug)
{
printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::remove(const uint32_t logic_block_id)
{
if (is_load_)
{
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
}
int ret = file_op_->munmap_file();
if (TFS_SUCCESS != ret)
{
return ret;
}
ret = file_op_->unlink_file();
return ret;
}
int IndexHandle::flush()
{
int ret = file_op_->flush_file();
if (TFS_SUCCESS != ret)
{
fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
}
return ret;
}
}
}
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->create(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "create index %d failed.\n", block_id);
delete index_handle;
exit(-3);
}
/* 2. 生成主块文件 */
stringstream tmp_stream;
tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
tmp_stream >> mainblock_path;
largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREAT);
ret = mainblock->ftruncate_file(main_blocksize);
if (ret != 0)
{
fprintf(stderr, "create main block %s failed. reason: %s\n", mainblock_path.c_str(), strerror(errno));
delete mainblock;
index_handle->remove(block_id);
exit(-2);
}
/* 其他操作 */
mainblock->close_file();
index_handle->flush();
delete mainblock;
delete index_handle;
return 0;
}
编译:
g++ block_init_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp mmap_file_op.cpp -o block_init_test
执行:
索引处理类 - 写文件流程实现(一)
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->load(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "load index %d failed.\n", block_id);
delete index_handle;
exit(-2);
}
/* 2. 写入文件到主块文件中 */
stringstream tmp_stream;
tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
tmp_stream >> mainblock_path;
largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREAT);
char buffer[4096];
memset(buffer, '6', sizeof(buffer));
int32_t data_offset = index_handle->get_block_data_offset();
uint32_t file_no = index_handle->block_info()->seq_no_;
if ((ret = mainblock->pwrite_file(buffer, sizeof(buffer), data_offset)) != TFS_SUCCESS)
{
fprintf(stderr, "write to main block failed. ret: %d, reason: %s \n", ret, strerror(errno));
mainblock->close_file();
delete mainblock;
delete index_handle;
exit(-3);
}
/* 3.索引文件中写入 Metainfo */
largefile::Metalnfo meta;
meta.set_file_id(file_no);
meta.set_offset(data_offset);
meta.set_size(sizeof(buffer));
delete mainblock;
delete index_handle;
return 0;
}
索引处理类 - 写文件流程实现(二)
#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_
#include "common.h"
#include "mmap_file_op.h"
namespace qiniu
{
namespace largefile
{
struct IndexHeader
{
public:
IndexHeader()
{
memset(this, 0, sizeof(IndexHeader));
}
Blockinfo block_info_; /* meta block info */
int32_t bucket_size_; /* hash bucket size */
int32_t data_file_offset_; /* offset to write next data in block */
int32_t index_file_size_; /* offset after index_header + all bucketa */
int32_t free_head_offset_; /* free meta node list, for reuse */
};
class IndexHandle
{
public:
IndexHandle(const std::string &base_path, const uint32_t main_block_id);
~IndexHandle();
int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
int load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
/* remove index unmmap and file */
int remove(const uint32_t logic_block_id);
int flush();
IndexHeader *index_header()
{
return reinterpret_cast<IndexHeader *>(file_op_->get_map_data());
}
Blockinfo *block_info()
{
return reinterpret_cast<Blockinfo *>(file_op_->get_map_data());
}
int32_t bucket_size() const
{
//return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->bucket_size;
reinterpret_cast<IndexHeader *>(file_op_->get_map_data())->bucket_size_;
}
int32_t get_block_data_offset() const
{
return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->data_file_offset_;
}
int write_segment_meta(const uint64_t key, Metalnfo &meta);
private:
MMapFileOperation *file_op_;
bool is_load_;
};
}
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */
#include "common.h"
#include "index_handle.h"
#include "sstream"
static int debug = 1;
namespace qiniu
{
namespace largefile
{
/* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
{
/* create file_op_ handle object */
std::stringstream tmp_stream;
tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */
std::string index_path;
tmp_stream >> index_path;
file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
is_load_ = false;
}
IndexHandle::~IndexHandle()
{
if (file_op_)
{
delete file_op_;
file_op_ = NULL;
}
}
int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
{
int ret = TFS_SUCCESS;
if (debug)
{
printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
}
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return TFS_ERROR;
}else if(file_size == 0) /* empty file */
{
IndexHeader i_header;
i_header.block_info_.block_id_ = logic_block_id;
i_header.block_info_.seq_no_ = 1;
i_header.bucket_size_ = bucket_size;
i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);
/* index header + total buckets */
char *init_data = new char[i_header.index_file_size_];
memcpy(init_data, &i_header, sizeof(IndexHandle));
memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));
/* write index header and buckets into file */
ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);
delete []init_data;
init_data = NULL;
if (ret != TFS_ERROR)
{
return ret;
}
ret = file_op_->flush_file();
if (ret != TFS_SUCCESS)
{
return ret;
}
}else /* file size > 0, index already exist */
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}
ret = file_op_->mmap_file(map_optiont);
if (ret != TFS_SUCCESS)
{
return ret;
}
is_load_ = true;
if (debug)
{
printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
{
int ret = TFS_SUCCESS;
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return file_size;
}
else if (file_size == 0) /* empty file */
{
return EXIT_INDEX_CORRUPT_ERROR;
}
MMapOption tmp_map_option = map_option;
if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
{
tmp_map_option.first_mmap_size_ = file_size;
}
ret = file_op_->mmap_file(tmp_map_option);
if (ret != TFS_SUCCESS)
{
return ret;
}
if (0 == bucket_size() || 0 == block_info()->block_id_)
{
fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check file size */
int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);
if (file_size < index_file_size)
{
fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check block_id_ size */
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
/* check bucket_size */
if (_bucket_size != bucket_size())
{
fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
return EXIT_BUCKET_CONFIGURE_ERROR;
}
is_load_ = true;
if (debug)
{
printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::remove(const uint32_t logic_block_id)
{
if (is_load_)
{
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
}
int ret = file_op_->munmap_file();
if (TFS_SUCCESS != ret)
{
return ret;
}
ret = file_op_->unlink_file();
return ret;
}
int IndexHandle::flush()
{
int ret = file_op_->flush_file();
if (TFS_SUCCESS != ret)
{
fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
}
return ret;
}
int IndexHandle::write_segment_meta(const uint64_t key, Metalnfo &meta)
{
int32_t current_offset = 0;
/* 思考? key 存在吗? 存在 => 处理? 不存在 => 处理? */
/* 1. 从文件哈希表中查找 key 是否存在. hash_find(key, current_offset, previous_offset */
/* 2. 不存在就写入 meta 到文件哈希表中 hash_insert(meta, slot, previous_offset) */
}
}
}
索引处理类 - 哈希查找实现
#include "common.h"
#include "index_handle.h"
#include "sstream"
static int debug = 1;
namespace qiniu
{
namespace largefile
{
/* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
{
/* create file_op_ handle object */
std::stringstream tmp_stream;
tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */
std::string index_path;
tmp_stream >> index_path;
file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
is_load_ = false;
}
IndexHandle::~IndexHandle()
{
if (file_op_)
{
delete file_op_;
file_op_ = NULL;
}
}
int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
{
int ret = TFS_SUCCESS;
if (debug)
{
printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
}
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return TFS_ERROR;
}else if(file_size == 0) /* empty file */
{
IndexHeader i_header;
i_header.block_info_.block_id_ = logic_block_id;
i_header.block_info_.seq_no_ = 1;
i_header.bucket_size_ = bucket_size;
i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);
/* index header + total buckets */
char *init_data = new char[i_header.index_file_size_];
memcpy(init_data, &i_header, sizeof(IndexHandle));
memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));
/* write index header and buckets into file */
ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);
delete []init_data;
init_data = NULL;
if (ret != TFS_ERROR)
{
return ret;
}
ret = file_op_->flush_file();
if (ret != TFS_SUCCESS)
{
return ret;
}
}else /* file size > 0, index already exist */
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}
ret = file_op_->mmap_file(map_optiont);
if (ret != TFS_SUCCESS)
{
return ret;
}
is_load_ = true;
if (debug)
{
printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
{
int ret = TFS_SUCCESS;
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return file_size;
}
else if (file_size == 0) /* empty file */
{
return EXIT_INDEX_CORRUPT_ERROR;
}
MMapOption tmp_map_option = map_option;
if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
{
tmp_map_option.first_mmap_size_ = file_size;
}
ret = file_op_->mmap_file(tmp_map_option);
if (ret != TFS_SUCCESS)
{
return ret;
}
if (0 == bucket_size() || 0 == block_info()->block_id_)
{
fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check file size */
int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);
if (file_size < index_file_size)
{
fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check block_id_ size */
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
/* check bucket_size */
if (_bucket_size != bucket_size())
{
fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
return EXIT_BUCKET_CONFIGURE_ERROR;
}
is_load_ = true;
if (debug)
{
printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::remove(const uint32_t logic_block_id)
{
if (is_load_)
{
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
}
int ret = file_op_->munmap_file();
if (TFS_SUCCESS != ret)
{
return ret;
}
ret = file_op_->unlink_file();
return ret;
}
int IndexHandle::flush()
{
int ret = file_op_->flush_file();
if (TFS_SUCCESS != ret)
{
fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
}
return ret;
}
int IndexHandle::write_segment_meta(const uint64_t key, Metalnfo &meta)
{
int32_t current_offset = 0;
/* 思考? key 存在吗? 存在 => 处理? 不存在 => 处理? */
/* 1. 从文件哈希表中查找 key 是否存在. hash_find(key, current_offset, previous_offset */
/* 2. 不存在就写入 meta 到文件哈希表中 hash_insert(meta, slot, previous_offset) */
}
int IndexHandle::hash_find(const uint64_t key, int32_t ¤t_offset, int32_t &previous_offset)
{
int ret = TFS_SUCCESS;
Metalnfo meta_info;
current_offset = 0;
previous_offset = 0;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = key % bucket_size();
/* 2. 读取桶首节点存储的第一个节点的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR */
/* 3. 根据偏移量读取存储的metainfo */
/* 4. 与key进行比较, 相等则设置 current_offset 和 previous_offset 并返回 TFS_SUCCESS, 否则继续执行5 */
/* 5. 从 metainfo 中取得下一个节点的在文件中的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR,
否则继续跳转到3循环执行 */
int32_t pos = bucket_size()[slot];
for (; pos != 0; )
{
ret = file_op_->pread_file(reinterpret_cast<char *>(meta_info)(&meta_info), sizeof(Metalnfo), pos);
if (TFS_SUCCESS != ret)
{
return ret;
}
if (hash_compare(key, meta_info.get_key()))
{
current_offset = pos;
return TFS_SUCCESS;
}
previous_offset = pos;
pos = meta_info.get_next_meta_offset();
}
return EXIT_META_NOT_FOUND_ERROR;
}
}
}
索引处理类 - 哈希插入实现
#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_
#include "common.h"
#include "mmap_file_op.h"
namespace qiniu
{
namespace largefile
{
struct IndexHeader
{
public:
IndexHeader()
{
memset(this, 0, sizeof(IndexHeader));
}
Blockinfo block_info_; /* meta block info */
int32_t bucket_size_; /* hash bucket size */
int32_t data_file_offset_; /* offset to write next data in block */
int32_t index_file_size_; /* offset after index_header + all bucketa */
int32_t free_head_offset_; /* free meta node list, for reuse */
};
class IndexHandle
{
public:
IndexHandle(const std::string &base_path, const uint32_t main_block_id);
~IndexHandle();
int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
int load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
/* remove index unmmap and file */
int remove(const uint32_t logic_block_id);
int flush();
IndexHeader *index_header()
{
return reinterpret_cast<IndexHeader *>(file_op_->get_map_data());
}
Blockinfo *block_info()
{
return reinterpret_cast<Blockinfo *>(file_op_->get_map_data());
}
int32_t bucket_size() const
{
//return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->bucket_size;
reinterpret_cast<IndexHeader *>(file_op_->get_map_data())->bucket_size_;
}
int32_t get_block_data_offset() const
{
return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->data_file_offset_;
}
int32_t write_segment_meta(const uint64_t key, Metalnfo &meta);
int32_t hash_find(const uint64_t key, int32_t ¤t_offset, int32_t &previous_offset); /* 哈希查找 */
int32_t hash_insert(const uint64_t key, int32_t previous_offset, Metalnfo &meta); /* 哈希插入 */
int32_t *bucket_slot()
{
reinterpret_cast<int32_t *>(reinterpret_cast<char *>(file_op_->get_map_data() + sizeof(IndexHeader)));
}
private:
bool hash_compare(cosnt uint64_t left_key, const uint64_t right_key)
{
return (left_key == right_key);
}
MMapFileOperation *file_op_;
bool is_load_;
};
}
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */
#include "common.h"
#include "index_handle.h"
#include "sstream"
static int debug = 1;
namespace qiniu
{
namespace largefile
{
/* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
{
/* create file_op_ handle object */
std::stringstream tmp_stream;
tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */
std::string index_path;
tmp_stream >> index_path;
file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
is_load_ = false;
}
IndexHandle::~IndexHandle()
{
if (file_op_)
{
delete file_op_;
file_op_ = NULL;
}
}
int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
{
int ret = TFS_SUCCESS;
if (debug)
{
printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
}
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return TFS_ERROR;
}else if(file_size == 0) /* empty file */
{
IndexHeader i_header;
i_header.block_info_.block_id_ = logic_block_id;
i_header.block_info_.seq_no_ = 1;
i_header.bucket_size_ = bucket_size;
i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);
/* index header + total buckets */
char *init_data = new char[i_header.index_file_size_];
memcpy(init_data, &i_header, sizeof(IndexHandle));
memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));
/* write index header and buckets into file */
ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);
delete []init_data;
init_data = NULL;
if (ret != TFS_ERROR)
{
return ret;
}
ret = file_op_->flush_file();
if (ret != TFS_SUCCESS)
{
return ret;
}
}else /* file size > 0, index already exist */
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}
ret = file_op_->mmap_file(map_optiont);
if (ret != TFS_SUCCESS)
{
return ret;
}
is_load_ = true;
if (debug)
{
printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
{
int ret = TFS_SUCCESS;
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return file_size;
}
else if (file_size == 0) /* empty file */
{
return EXIT_INDEX_CORRUPT_ERROR;
}
MMapOption tmp_map_option = map_option;
if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
{
tmp_map_option.first_mmap_size_ = file_size;
}
ret = file_op_->mmap_file(tmp_map_option);
if (ret != TFS_SUCCESS)
{
return ret;
}
if (0 == bucket_size() || 0 == block_info()->block_id_)
{
fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check file size */
int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);
if (file_size < index_file_size)
{
fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check block_id_ size */
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
/* check bucket_size */
if (_bucket_size != bucket_size())
{
fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
return EXIT_BUCKET_CONFIGURE_ERROR;
}
is_load_ = true;
if (debug)
{
printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::remove(const uint32_t logic_block_id)
{
if (is_load_)
{
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
}
int ret = file_op_->munmap_file();
if (TFS_SUCCESS != ret)
{
return ret;
}
ret = file_op_->unlink_file();
return ret;
}
int IndexHandle::flush()
{
int ret = file_op_->flush_file();
if (TFS_SUCCESS != ret)
{
fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
}
return ret;
}
int IndexHandle::write_segment_meta(const uint64_t key, Metalnfo &meta)
{
int32_t current_offset = 0, previous_offset = 0;
/* 思考? key 存在吗? 存在 => 处理? 不存在 => 处理? */
/* 1. 从文件哈希表中查找 key 是否存在. hash_find(key, current_offset, previous_offset */
int ret = hash_find(key, current_offset, previous_offset);
if (TFS_SUCCESS == ret)
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}else if (EXIT_META_NOT_FOUND_ERROR == ret)
{
return ret;
}
/* 2. 不存在就写入 meta 到文件哈希表中 hash_insert(key, previous_offset, meta) */
ret = hash_insert(key, previous_offset, meta);
return ret;
}
int IndexHandle::hash_find(const uint64_t key, int32_t ¤t_offset, int32_t &previous_offset)
{
int ret = TFS_SUCCESS;
Metalnfo meta_info;
current_offset = 0;
previous_offset = 0;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = (int32_t)key % bucket_size();
/* 2. 读取桶首节点存储的第一个节点的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR */
/* 3. 根据偏移量读取存储的metainfo */
/* 4. 与key进行比较, 相等则设置 current_offset 和 previous_offset 并返回 TFS_SUCCESS, 否则继续执行5 */
/* 5. 从 metainfo 中取得下一个节点的在文件中的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR,
否则继续跳转到3循环执行 */
int32_t pos = bucket_size()[slot];
for (; pos != 0; )
{
ret = file_op_->pread_file(reinterpret_cast<char *>(meta_info)(&meta_info), sizeof(Metalnfo), pos);
if (TFS_SUCCESS != ret)
{
return ret;
}
if (hash_compare(key, meta_info.get_key()))
{
current_offset = pos;
return TFS_SUCCESS;
}
previous_offset = pos;
pos = meta_info.get_next_meta_offset();
}
return EXIT_META_NOT_FOUND_ERROR;
}
int32_t IndexHandle::hash_insert(const uint64_t key, int32_t previous_offset, Metalnfo &meta)
{
Metalnfo tmp_meta_info;
int ret = TFS_SUCCESS;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = static_cast<uint32_t>(key) % bucket_size();
/* 2. 确定 meta 节点存储中文件中的偏移量 */
int32_t current_offset = index_header()->index_file_size_;
index_header()->index_file_size_ += sizeof(Metalnfo);
/* 3. 将 meta 节点写入索引文件中 */
meta.set_next_meta_offset(0); /* 我怀疑我忘了实现 */
ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&meta), sizeof(Metalnfo), current_offset);
if (ret != TFS_SUCCESS
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
/* 4. 将 meta 节点插入到哈希链表中 */
/* 当前一个节点已经存在 */
if (0 != previous_offset)
{
ret = file_op_->pread_file(reinterpret_cast<char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
if (TFS_SUCCESS != ret)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
tmp_meta_info.set_next_meta_offset(current_offset);
ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
if (TFS_SUCCESS != ret)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
}
else /* 不存在前一个节点的情况 */
{
bucket_slot()[slot] = current_offset;
}
return TFS_SUCCESS;
}
}
}
索引处理类 - 更新块信息
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->load(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "load index %d failed.\n", block_id);
delete index_handle;
exit(-2);
}
/* 2. 写入文件到主块文件中 */
stringstream tmp_stream;
tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
tmp_stream >> mainblock_path;
largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREAT);
char buffer[4096];
memset(buffer, '6', sizeof(buffer));
int32_t data_offset = index_handle->get_block_data_offset();
uint32_t file_no = index_handle->block_info()->seq_no_;
if ((ret = mainblock->pwrite_file(buffer, sizeof(buffer), data_offset)) != TFS_SUCCESS)
{
fprintf(stderr, "write to main block failed. ret: %d, reason: %s \n", ret, strerror(errno));
mainblock->close_file();
delete mainblock;
delete index_handle;
exit(-3);
}
/* 3.索引文件中写入 Metainfo */
largefile::Metalnfo meta;
meta.set_file_id(file_no);
meta.set_offset(data_offset);
meta.set_size(sizeof(buffer));
ret = index_handle->write_segment_meta(meta.get_key(), meta);
if (ret == largefile::TFS_SUCCESS)
{
/* 1. 更新索引头部信息 */
index_handle->commit_block_offset(sizeof(buffer));
/* 2. 更新块信息 */
index_handle->commit_block_offset(C_OPER_INSERT, sizeof(buffer));
ret = index_handle->flush();
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "flush mainblock %d failed file no: %u \n", block_id_, file_no);
}
}
else
{
fprintf(stderr, "write_segement_meta - mainblock %d failed. file no: %u \n", block_id_, file_no);
}
if (ret != TFS_SUCCESS)
{
fprintf(stderr, "write to mainblock %d failed no: %u \n", block_id_, file_no);
}
else
{
if (debug)
{
printf("write successfully. file no: %u, block_id: %d \n", file_no, block_id_);
}
}
mainblock->close_file();
delete mainblock;
delete index_handle;
return 0;
}
#ifndef QINIU_LARGEFILE_INDEX_HANDLE_H_
#define QINIU_LARGEFILE_INDEX_HANDLE_H_
#include "common.h"
#include "mmap_file_op.h"
namespace qiniu
{
namespace largefile
{
struct IndexHeader
{
public:
IndexHeader()
{
memset(this, 0, sizeof(IndexHeader));
}
Blockinfo block_info_; /* meta block info */
int32_t bucket_size_; /* hash bucket size */
int32_t data_file_offset_; /* offset to write next data in block */
int32_t index_file_size_; /* offset after index_header + all bucketa */
int32_t free_head_offset_; /* free meta node list, for reuse */
};
class IndexHandle
{
public:
IndexHandle(const std::string &base_path, const uint32_t main_block_id);
~IndexHandle();
int create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
int load(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont);
/* remove index unmmap and file */
int remove(const uint32_t logic_block_id);
int flush();
IndexHeader *index_header()
{
return reinterpret_cast<IndexHeader *>(file_op_->get_map_data());
}
Blockinfo *block_info()
{
return reinterpret_cast<Blockinfo *>(file_op_->get_map_data());
}
int32_t bucket_size() const
{
//return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->bucket_size;
reinterpret_cast<IndexHeader *>(file_op_->get_map_data())->bucket_size_;
}
int32_t get_block_data_offset() const
{
return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->data_file_offset_;
}
void commit_block_offset(const int file_size)
{
return reinterpret_cast<IndexHandle *>(file_op_->get_map_data())->data_file_offset_ += file_size;
}
int32_t write_segment_meta(const uint64_t key, Metalnfo &meta);
int32_t hash_find(const uint64_t key, int32_t ¤t_offset, int32_t &previous_offset); /* 哈希查找 */
int32_t hash_insert(const uint64_t key, int32_t previous_offset, Metalnfo &meta); /* 哈希插入 */
int32_t *bucket_slot()
{
reinterpret_cast<int32_t *>(reinterpret_cast<char *>(file_op_->get_map_data() + sizeof(IndexHeader)));
}
int update_block_info(const OperType oper_type, const uint32_t modify_size); /* 更新块信息 */
private:
bool hash_compare(cosnt uint64_t left_key, const uint64_t right_key)
{
return (left_key == right_key);
}
MMapFileOperation *file_op_;
bool is_load_;
};
}
}
#endif /* QINIU_LARGEFILE_INDEX_HANDLE_H_ */
#include "common.h"
#include "index_handle.h"
#include "sstream"
static int debug = 1;
namespace qiniu
{
namespace largefile
{
/* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
{
/* create file_op_ handle object */
std::stringstream tmp_stream;
tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */
std::string index_path;
tmp_stream >> index_path;
file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
is_load_ = false;
}
IndexHandle::~IndexHandle()
{
if (file_op_)
{
delete file_op_;
file_op_ = NULL;
}
}
int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
{
int ret = TFS_SUCCESS;
if (debug)
{
printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
}
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return TFS_ERROR;
}else if(file_size == 0) /* empty file */
{
IndexHeader i_header;
i_header.block_info_.block_id_ = logic_block_id;
i_header.block_info_.seq_no_ = 1;
i_header.bucket_size_ = bucket_size;
i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);
/* index header + total buckets */
char *init_data = new char[i_header.index_file_size_];
memcpy(init_data, &i_header, sizeof(IndexHandle));
memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));
/* write index header and buckets into file */
ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);
delete []init_data;
init_data = NULL;
if (ret != TFS_ERROR)
{
return ret;
}
ret = file_op_->flush_file();
if (ret != TFS_SUCCESS)
{
return ret;
}
}else /* file size > 0, index already exist */
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}
ret = file_op_->mmap_file(map_optiont);
if (ret != TFS_SUCCESS)
{
return ret;
}
is_load_ = true;
if (debug)
{
printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
{
int ret = TFS_SUCCESS;
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return file_size;
}
else if (file_size == 0) /* empty file */
{
return EXIT_INDEX_CORRUPT_ERROR;
}
MMapOption tmp_map_option = map_option;
if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
{
tmp_map_option.first_mmap_size_ = file_size;
}
ret = file_op_->mmap_file(tmp_map_option);
if (ret != TFS_SUCCESS)
{
return ret;
}
if (0 == bucket_size() || 0 == block_info()->block_id_)
{
fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check file size */
int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);
if (file_size < index_file_size)
{
fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check block_id_ size */
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
/* check bucket_size */
if (_bucket_size != bucket_size())
{
fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
return EXIT_BUCKET_CONFIGURE_ERROR;
}
is_load_ = true;
if (debug)
{
printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::remove(const uint32_t logic_block_id)
{
if (is_load_)
{
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
}
int ret = file_op_->munmap_file();
if (TFS_SUCCESS != ret)
{
return ret;
}
ret = file_op_->unlink_file();
return ret;
}
int IndexHandle::flush()
{
int ret = file_op_->flush_file();
if (TFS_SUCCESS != ret)
{
fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
}
return ret;
}
int IndexHandle::write_segment_meta(const uint64_t key, Metalnfo &meta)
{
int32_t current_offset = 0, previous_offset = 0;
/* 思考? key 存在吗? 存在 => 处理? 不存在 => 处理? */
/* 1. 从文件哈希表中查找 key 是否存在. hash_find(key, current_offset, previous_offset */
int ret = hash_find(key, current_offset, previous_offset);
if (TFS_SUCCESS == ret)
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}else if (EXIT_META_NOT_FOUND_ERROR == ret)
{
return ret;
}
/* 2. 不存在就写入 meta 到文件哈希表中 hash_insert(key, previous_offset, meta) */
ret = hash_insert(key, previous_offset, meta);
return ret;
}
int IndexHandle::hash_find(const uint64_t key, int32_t ¤t_offset, int32_t &previous_offset)
{
int ret = TFS_SUCCESS;
Metalnfo meta_info;
current_offset = 0;
previous_offset = 0;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = (int32_t)key % bucket_size();
/* 2. 读取桶首节点存储的第一个节点的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR */
/* 3. 根据偏移量读取存储的metainfo */
/* 4. 与key进行比较, 相等则设置 current_offset 和 previous_offset 并返回 TFS_SUCCESS, 否则继续执行5 */
/* 5. 从 metainfo 中取得下一个节点的在文件中的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR,
否则继续跳转到3循环执行 */
int32_t pos = bucket_size()[slot];
for (; pos != 0; )
{
ret = file_op_->pread_file(reinterpret_cast<char *>(meta_info)(&meta_info), sizeof(Metalnfo), pos);
if (TFS_SUCCESS != ret)
{
return ret;
}
if (hash_compare(key, meta_info.get_key()))
{
current_offset = pos;
return TFS_SUCCESS;
}
previous_offset = pos;
pos = meta_info.get_next_meta_offset();
}
return EXIT_META_NOT_FOUND_ERROR;
}
int32_t IndexHandle::hash_insert(const uint64_t key, int32_t previous_offset, Metalnfo &meta)
{
Metalnfo tmp_meta_info;
int ret = TFS_SUCCESS;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = static_cast<uint32_t>(key) % bucket_size();
/* 2. 确定 meta 节点存储中文件中的偏移量 */
int32_t current_offset = index_header()->index_file_size_;
index_header()->index_file_size_ += sizeof(Metalnfo);
/* 3. 将 meta 节点写入索引文件中 */
meta.set_next_meta_offset(0); /* 我怀疑我忘了实现 */
ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&meta), sizeof(Metalnfo), current_offset);
if (ret != TFS_SUCCESS
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
/* 4. 将 meta 节点插入到哈希链表中 */
/* 当前一个节点已经存在 */
if (0 != previous_offset)
{
ret = file_op_->pread_file(reinterpret_cast<char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
if (TFS_SUCCESS != ret)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
tmp_meta_info.set_next_meta_offset(current_offset);
ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
if (TFS_SUCCESS != ret)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
}
else /* 不存在前一个节点的情况 */
{
bucket_slot()[slot] = current_offset;
}
return TFS_SUCCESS;
}
int IndexHandle::update_block_info(const OperType oper_type, const uint32_t modify_size) /* 更新块信息 */
{
if (block_info()->block_id_ == 0)
{
return EXIT_BLOCKID_ZERO_ERROR;
}
if (oper_type == C_OPER_INSERT)
{
++block_info()->version_;
++block_info()->file_count_;
++block_info()->seq_no_;
block_info()->size_ += modify_size;
}
if (debug)
{
printf("updata block info. blockid: %u, version: %u, file count: %u, size: %u, del file count: %u, del size: %u, seq no: %u, oper type: %d\n",
block_info()->block_id_, block_info()->version_, block_info()->file_count_, block_info()->size_,
block_info()->del_file_count_, block_info()->del_size_, block_info()->seq_no_, oper_type);
}
return TFS_SUCCESS;
}
}
}
写流程实现 - 单元测试
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->load(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "load index %d failed.\n", block_id);
delete index_handle;
exit(-2);
}
/* 2. 写入文件到主块文件中 */
stringstream tmp_stream;
tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
tmp_stream >> mainblock_path;
largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR | O_LARGEFILE | O_CREAT);
char buffer[4096];
memset(buffer, '6', sizeof(buffer));
int32_t data_offset = index_handle->get_block_data_offset();
uint32_t file_no = index_handle->block_info()->seq_no_;
if ((ret = mainblock->pwrite_file(buffer, sizeof(buffer), data_offset)) != largefile::TFS_SUCCESS)
{
fprintf(stderr, "write to main block failed. ret: %d, reason: %s \n", ret, strerror(errno));
mainblock->close_file();
delete mainblock;
delete index_handle;
exit(-3);
}
/* 3.索引文件中写入 Metainfo */
largefile::Metalnfo meta;
meta.set_file_id(file_no);
meta.set_offset(data_offset);
meta.set_size(sizeof(buffer));
ret = index_handle->write_segment_meta(meta.get_key(), meta);
if (ret == largefile::TFS_SUCCESS)
{
/* 1. 更新索引头部信息 */
index_handle->commit_block_offset(sizeof(buffer));
/* 2. 更新块信息 */
index_handle->update_block_info(largefile::C_OPER_INSERT, sizeof(buffer));
ret = index_handle->flush();
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "flush mainblock %d failed file no: %u \n", block_id, file_no);
}
}
else
{
fprintf(stderr, "write_segement_meta - mainblock %d failed. file no: %u \n", block_id, file_no);
}
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "write to mainblock %d failed no: %u \n", block_id, file_no);
}
else
{
if (debug)
{
printf("write successfully. file no: %u, block_id: %d \n", file_no, block_id);
}
}
mainblock->close_file();
delete mainblock;
delete index_handle;
return 0;
}
编译:
g++ block_write_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp mmap_file_op.cpp -o block_write_test
执行:
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->load(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "load index %d failed.\n", block_id);
delete index_handle;
exit(-2);
}
delete index_handle;
return 0;
}
编译:
g++ block_stat.cpp file_op.cpp index_handle.cpp mmap_file.cpp mmap_file_op.cpp -o block_stat
执行:
结语:
调试调了好久! ! !
代码是跟着老师抄的, 思想也是老师的,(看你能学多少咯) 还是好好复习. 可能代码写多才有感觉吧? 谁知道呢? 总之, 社会推着你必须学(学习就对了).
-量变引起质变-
时间: 2020-06-29
951
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
