索引处理类 - 读流程实现
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->load(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "load index %d failed.\n", block_id);
delete index_handle;
exit(-2);
}
/* 2. 读取文件的 meta info */
uint64_t file_id = 0;
cout << "Type your file_id: "<<endl;
cin >> file_id;
if (file_id < 1)
{
cerr << "Invalid fileid, exit. " <<endl;
}
largefile::Metalnfo meta;
ret = index_handle->read_segment_meta(file_id, meta); /* read_segment_meta */
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "read_segment_meta error. file_id: %lu, ret: %d\n", file_id, ret);
exit(-3);
}
/* 3. 根据meta info 读取文件 */
stringstream tmp_stream;
tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
tmp_stream >> mainblock_path;
largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR);
char *buffer = new char[meta.get_size() + 1];
ret = mainblock->pread_file(buffer, meta.get_size(), meta.get_offset());
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "read from main block failed. ret: %d, reason: %s \n", ret, strerror(errno));
mainblock->close_file();
delete mainblock;
delete index_handle;
exit(-3);
}
buffer[meta.get_size()] = '\0';
printf("read: %d: content: %s \n", meta.get_size(), buffer);
mainblock->close_file();
delete mainblock;
delete index_handle;
return 0;
}
编译:
g++ block_read_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp mmap_file_op.cpp -o block_read_test
执行(很明显错误了, 但是我并不知道, 因为这是别人的思想, 我需要反复看 UML 图, 和了解它的思想才能解决):
索引处理类 - 删除流程实现
#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"
using namespace std;
using namespace qiniu;
const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;
int main(int argc, char *argv[])
{
string mainblock_path;
string index_path;
int32_t ret = largefile::TFS_SUCCESS;
cout << "Type your bockid :" <<endl;
cin >> block_id;
if (block_id < 1)
{
cerr << "Invalid bockid, exit." <<endl;
exit(-1);
}
/* 1. 创建索引文件 */
largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);
if (debug)
{
printf("Init index ...\n");
}
ret = index_handle->load(block_id, bucket_size, mmap_option);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "load index %d failed.\n", block_id);
delete index_handle;
exit(-2);
}
/* 2. 删除指定文件的 meta info */
uint64_t file_id = 0;
cout << "Type your file_id: "<<endl;
cin >> file_id;
if (file_id < 1)
{
cerr << "Invalid fileid, exit. " <<endl;
}
largefile::Metalnfo meta;
ret = index_handle->read_segment_meta(file_id, meta); /* read_segment_meta */
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "read_segment_meta error. file_id: %lu, ret: %d\n", file_id, ret);
exit(-3);
}
ret = index_handle->delete_segment_meta(file_id);
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "delete index failed. file_id: %lu. ret: %d \n", file_id, ret);
}
ret = index_handle->flush();
if (ret != largefile::TFS_SUCCESS)
{
fprintf(stderr, "flush mainblock %d failed file no: %lu \n", block_id, file_id);
exit(-4);
}
printf("delete successfully ! \n");
delete index_handle;
return 0;
}
编译:
g++ block_delete_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp mmap_file_op.cpp -o block_delete_test
执行:
(我人傻了, 可能手抖或走神写错代码, 完全测试失败! 别问我错误原因, 我不知道, 因为这代码是别人的思想, 我只是刚接触, 不过没有关系, 第一轮我也是有目的的-- 就是看到思想, 和为啥它这么写, 第二次再来, 我觉得就没问题)
可重用 meta 节点实现
#include "common.h"
#include "index_handle.h"
#include "sstream"
static int debug = 1;
namespace qiniu
{
namespace largefile
{
/* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
{
/* create file_op_ handle object */
std::stringstream tmp_stream;
tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */
std::string index_path;
tmp_stream >> index_path;
file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
is_load_ = false;
}
IndexHandle::~IndexHandle()
{
if (file_op_)
{
delete file_op_;
file_op_ = NULL;
}
}
int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
{
int ret = TFS_SUCCESS;
if (debug)
{
printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
}
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return TFS_ERROR;
}else if(file_size == 0) /* empty file */
{
IndexHeader i_header;
i_header.block_info_.block_id_ = logic_block_id;
i_header.block_info_.seq_no_ = 1;
i_header.bucket_size_ = bucket_size;
i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);
/* index header + total buckets */
char *init_data = new char[i_header.index_file_size_];
memcpy(init_data, &i_header, sizeof(IndexHandle));
memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));
/* write index header and buckets into file */
ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);
delete []init_data;
init_data = NULL;
if (ret != TFS_ERROR)
{
return ret;
}
ret = file_op_->flush_file();
if (ret != TFS_SUCCESS)
{
return ret;
}
}else /* file size > 0, index already exist */
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}
ret = file_op_->mmap_file(map_optiont);
if (ret != TFS_SUCCESS)
{
return ret;
}
is_load_ = true;
if (debug)
{
printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
{
int ret = TFS_SUCCESS;
if (is_load_)
{
return EXIT_INDEX_ALREADY_LOADED_ERROR;
}
int64_t file_size = file_op_->get_file_size();
if (file_size < 0)
{
return file_size;
}
else if (file_size == 0) /* empty file */
{
return EXIT_INDEX_CORRUPT_ERROR;
}
MMapOption tmp_map_option = map_option;
if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
{
tmp_map_option.first_mmap_size_ = file_size;
}
ret = file_op_->mmap_file(tmp_map_option);
if (ret != TFS_SUCCESS)
{
return ret;
}
if (debug)
{
printf("InexHandle::load - bucket size(): %d, index_header()->bucket_size_: %d, block id: %d \n", bucket_size(), index_header()->bucket_size_, block_info()->block_id_);
}
if (0 == bucket_size() || 0 == block_info()->block_id_)
{
fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check file size */
int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);
if (file_size < index_file_size)
{
fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
return EXIT_INDEX_CORRUPT_ERROR;
}
/* check block_id_ size */
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
/* check bucket_size */
if (_bucket_size != bucket_size())
{
fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
return EXIT_BUCKET_CONFIGURE_ERROR;
}
is_load_ = true;
if (debug)
{
printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n",
logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
}
return TFS_SUCCESS;
}
int IndexHandle::remove(const uint32_t logic_block_id)
{
if (is_load_)
{
if (logic_block_id != block_info()->block_id_)
{
fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
return EXIT_BLOCKID_CONFLICT_ERROR;
}
}
int ret = file_op_->munmap_file();
if (TFS_SUCCESS != ret)
{
return ret;
}
ret = file_op_->unlink_file();
return ret;
}
int IndexHandle::flush()
{
int ret = file_op_->flush_file();
if (TFS_SUCCESS != ret)
{
fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
}
return ret;
}
int IndexHandle::write_segment_meta(const uint64_t key, Metalnfo &meta)
{
int32_t current_offset = 0, previous_offset = 0;
/* 思考? key 存在吗? 存在 => 处理? 不存在 => 处理? */
/* 1. 从文件哈希表中查找 key 是否存在. hash_find(key, current_offset, previous_offset */
int ret = hash_find(key, current_offset, previous_offset);
if (TFS_SUCCESS == ret)
{
return EXIT_META_UNEXPECT_FOUND_ERROR;
}else if (EXIT_META_NOT_FOUND_ERROR == ret)
{
return ret;
}
/* 2. 不存在就写入 meta 到文件哈希表中 hash_insert(key, previous_offset, meta) */
ret = hash_insert(key, previous_offset, meta);
return ret;
}
int32_t IndexHandle::read_segment_meta(const uint64_t key, Metalnfo &meta)
{
int32_t current_offset = 0, previous_offset = 0;
/* 1. 确定 key 存放的桶(slot)的位置 */
//int32_t slot = static_cast<uint32_t>(key) % bucket_size();
int32_t ret = hash_find(key, current_offset, previous_offset);
if (ret == TFS_SUCCESS) /* exist */
{
ret = file_op_->pread_file(reinterpret_cast<char *>(&meta), sizeof(Metalnfo), current_offset);
return ret;
}
else
{
return ret;
}
}
int32_t IndexHandle::delete_segment_meta(const uint64_t key) /* 删除 */
{
int32_t current_offset = 0, previous_offset = 0;
int32_t ret = hash_find(key, current_offset, previous_offset);
if (ret != TFS_SUCCESS)
{
return ret;
}
Metalnfo meta_info;
ret = file_op_->pread_file(reinterpret_cast<char *>(&meta_info), sizeof(Metalnfo), current_offset);
if (ret != TFS_SUCCESS)
{
return ret;
}
int32_t next_pos = meta_info.get_next_meta_offset();
if (previous_offset == 0)
{
int32_t slot = static_cast<uint32_t>(key) % bucket_size();
bucket_slot()[slot] = next_pos;
}
else
{
Metalnfo pre_meta_info;
ret = file_op_->pread_file(reinterpret_cast<char *>(&pre_meta_info), sizeof(Metalnfo), previous_offset);
if (ret != TFS_SUCCESS)
{
return ret;
}
pre_meta_info.set_next_meta_offset(next_pos);
ret = file_op_->pwrite_file(reinterpret_cast<char *>(&pre_meta_info), sizeof(Metalnfo), previous_offset);
if (ret != TFS_SUCCESS)
{
return ret;
}
}
/* 把删除节点加入可重用节点链表(下一个小节实现) */
meta_info.set_next_meta_offset(free_head_offset()); /* index_header()->free_head_offset_ */
ret = file_op_->pwrite_file(reinterpret_cast<char *>(&meta_info), sizeof(Metalnfo), current_offset);
if (ret != TFS_SUCCESS)
{
return ret;
}
index_header()->free_head_offset_ = current_offset;
if (debug)
{
printf("delete_segment_meta - reuse metainfo current_offset: %d\n", current_offset);
}
update_block_info(C_OPER_DELETE, meta_info.get_size());
return TFS_SUCCESS;
}
int IndexHandle::hash_find(const uint64_t key, int32_t ¤t_offset, int32_t &previous_offset)
{
int ret = TFS_SUCCESS;
Metalnfo meta_info;
current_offset = 0;
previous_offset = 0;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = (int32_t)key % bucket_size();
/* 2. 读取桶首节点存储的第一个节点的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR */
/* 3. 根据偏移量读取存储的metainfo */
/* 4. 与key进行比较, 相等则设置 current_offset 和 previous_offset 并返回 TFS_SUCCESS, 否则继续执行5 */
/* 5. 从 metainfo 中取得下一个节点的在文件中的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR,
否则继续跳转到3循环执行 */
int32_t pos = bucket_slot()[slot];
for (; pos != 0; )
{
ret = file_op_->pread_file(reinterpret_cast<char *>(&meta_info), sizeof(Metalnfo), pos);
if (TFS_SUCCESS != ret)
{
return ret;
}
if (hash_compare(key, meta_info.get_key()))
{
current_offset = pos;
return TFS_SUCCESS;
}
previous_offset = pos;
pos = meta_info.get_next_meta_offset();
}
return EXIT_META_NOT_FOUND_ERROR;
}
int32_t IndexHandle::hash_insert(const uint64_t key, int32_t previous_offset, Metalnfo &meta)
{
Metalnfo tmp_meta_info;
int ret = TFS_SUCCESS;
int32_t current_offset = 0;
/* 1. 确定 key 存放的桶(slot)的位置 */
int32_t slot = static_cast<uint32_t>(key) % bucket_size();
/* 2. 确定 meta 节点存储中文件中的偏移量 */
if (free_head_offset() != 0)
{
ret = file_op_->pread_file(reinterpret_cast<char *>(&tmp_meta_info), sizeof(Metalnfo), free_head_offset());
if (ret != TFS_SUCCESS)
{
return ret;
}
current_offset = index_header()->free_head_offset_;
if (debug)
{
printf("reuse metainfo current_offset: %d\n", current_offset);
}
index_header()->free_head_offset_ = tmp_meta_info.get_next_meta_offset();
}
else
{
current_offset = index_header()->index_file_size_;
index_header()->index_file_size_ += sizeof(Metalnfo);
}
/* 3. 将 meta 节点写入索引文件中 */
meta.set_next_meta_offset(0); /* 我怀疑我忘了实现 */
ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&meta), sizeof(Metalnfo), current_offset);
if (ret != TFS_SUCCESS)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
/* 4. 将 meta 节点插入到哈希链表中 */
/* 当前一个节点已经存在 */
if (0 != previous_offset)
{
ret = file_op_->pread_file(reinterpret_cast<char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
if (TFS_SUCCESS != ret)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
tmp_meta_info.set_next_meta_offset(current_offset);
ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
if (TFS_SUCCESS != ret)
{
index_header()->index_file_size_ -= sizeof(Metalnfo);
return ret;
}
}
else /* 不存在前一个节点的情况 */
{
bucket_slot()[slot] = current_offset;
}
return TFS_SUCCESS;
}
int IndexHandle::update_block_info(const OperType oper_type, const uint32_t modify_size) /* 更新块信息 */
{
if (block_info()->block_id_ == 0)
{
return EXIT_BLOCKID_ZERO_ERROR;
}
if (oper_type == C_OPER_INSERT)
{
++block_info()->version_;
++block_info()->file_count_;
++block_info()->seq_no_;
block_info()->size_ += modify_size;
}
else if (oper_type == C_OPER_DELETE)
{
++block_info()->version_;
--block_info()->file_count_;
block_info()->size_ -= modify_size;
++block_info()->del_file_count_;
block_info()->del_size_ += modify_size;
}
if (debug)
{
printf("updata block info. blockid: %u, version: %u, file count: %u, size: %u, del file count: %u, del size: %u, seq no: %u, oper type: %d\n",
block_info()->block_id_, block_info()->version_, block_info()->file_count_, block_info()->size_,
block_info()->del_file_count_, block_info()->del_size_, block_info()->seq_no_, oper_type);
}
return TFS_SUCCESS;
}
}
}
编译:
g++ block_delete_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp mmap_file_op.cpp -o block_delete_test
执行:
(我人傻了, 可能手抖或走神写错代码, 完全测试失败! 别问我错误原因, 我不知道, 因为这代码是别人的思想, 不过没有关系, 第一轮我也是有目的的-- 就是看到思想, 和为啥它这么写, 第二轮复习再来, 我觉得就没问题)
结束 - 文件系统核心存储引擎
结语:
总之: 第一轮了解它的思想, 编程的核心就是思想 - > 再实现
最后上传老师的代码如下:
https://download.csdn.net/download/m0_45867846/12564372
时间: 2020-06-30