结束 - 文件系统核心存储引擎_实现(六)

索引处理类 - 读流程实现

在这里插入图片描述

#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"

using namespace std;
using namespace qiniu;

const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;

int main(int argc, char *argv[])
{
    string mainblock_path;
    string index_path;
    int32_t ret = largefile::TFS_SUCCESS;

    cout << "Type your bockid :" <<endl;
    cin >> block_id;

    if (block_id < 1)
    {
        cerr << "Invalid bockid, exit." <<endl;
        exit(-1);
    }


    /* 1. 创建索引文件 */
    largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);

    if (debug)
    {
        printf("Init index ...\n");
    }

    ret = index_handle->load(block_id, bucket_size, mmap_option);

    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "load index %d failed.\n", block_id);
        delete index_handle;
        exit(-2);
    }


    /* 2. 读取文件的 meta info */
    uint64_t file_id = 0;
    cout << "Type your file_id: "<<endl;
    cin >> file_id;

    if (file_id < 1)
    {
        cerr << "Invalid fileid, exit. " <<endl;
    }

    largefile::Metalnfo meta;

    ret = index_handle->read_segment_meta(file_id, meta); /* read_segment_meta */
    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "read_segment_meta error. file_id: %lu, ret: %d\n", file_id, ret);
        exit(-3);
    }


    /* 3. 根据meta info 读取文件 */
    stringstream tmp_stream;
    tmp_stream << "." << largefile::MAINBLOCK_DIR_PREFIX << block_id;
    tmp_stream >> mainblock_path;

    largefile::FileOperation *mainblock = new largefile::FileOperation(mainblock_path, O_RDWR);
    char *buffer = new char[meta.get_size() + 1];

    ret = mainblock->pread_file(buffer, meta.get_size(), meta.get_offset());
    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "read from main block failed. ret: %d, reason: %s \n", ret, strerror(errno));
        mainblock->close_file();

        delete mainblock;
        delete index_handle;
        exit(-3);
    }

    buffer[meta.get_size()] = '\0';
    printf("read: %d: content: %s \n", meta.get_size(), buffer);

    mainblock->close_file();
    delete mainblock;
    delete index_handle;

    return 0;
}

编译:

g++ block_read_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp  mmap_file_op.cpp -o block_read_test

执行(很明显错误了, 但是我并不知道, 因为这是别人的思想, 我需要反复看 UML 图, 和了解它的思想才能解决):

在这里插入图片描述

索引处理类 - 删除流程实现

在这里插入图片描述

#include "common.h"
#include "file_op.h"
#include "index_handle.h"
#include "sstream"

using namespace std;
using namespace qiniu;

const static largefile::MMapOption mmap_option = {1024000, 4096, 4096}; /* 内存映射的参数 */
const static uint32_t main_blocksize = 1024 * 1024 * 64; /* 主块文件的大小 */
const static uint32_t bucket_size = 1000; /* 哈希桶的大小 */
static int32_t block_id = 1;
static int debug = 1;

int main(int argc, char *argv[])
{
    string mainblock_path;
    string index_path;
    int32_t ret = largefile::TFS_SUCCESS;

    cout << "Type your bockid :" <<endl;
    cin >> block_id;

    if (block_id < 1)
    {
        cerr << "Invalid bockid, exit." <<endl;
        exit(-1);
    }


    /* 1. 创建索引文件 */
    largefile::IndexHandle *index_handle = new largefile::IndexHandle(".", block_id);

    if (debug)
    {
        printf("Init index ...\n");
    }

    ret = index_handle->load(block_id, bucket_size, mmap_option);

    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "load index %d failed.\n", block_id);
        delete index_handle;
        exit(-2);
    }


    /* 2. 删除指定文件的 meta info */
    uint64_t file_id = 0;
    cout << "Type your file_id: "<<endl;
    cin >> file_id;

    if (file_id < 1)
    {
        cerr << "Invalid fileid, exit. " <<endl;
    }

    largefile::Metalnfo meta;

    ret = index_handle->read_segment_meta(file_id, meta); /* read_segment_meta */
    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "read_segment_meta error. file_id: %lu, ret: %d\n", file_id, ret);
        exit(-3);
    }

    ret = index_handle->delete_segment_meta(file_id);
    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "delete index failed. file_id: %lu. ret: %d \n", file_id, ret);
    }

    ret = index_handle->flush(); 

    if (ret != largefile::TFS_SUCCESS)
    {
        fprintf(stderr, "flush mainblock %d failed file no: %lu \n", block_id, file_id);
        exit(-4);
    }


    printf("delete successfully ! \n");
    delete index_handle;

    return 0;
}

编译:

g++ block_delete_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp  mmap_file_op.cpp -o block_delete_test

执行:

(我人傻了, 可能手抖或走神写错代码, 完全测试失败! 别问我错误原因, 我不知道, 因为这代码是别人的思想, 我只是刚接触, 不过没有关系, 第一轮我也是有目的的-- 就是看到思想, 和为啥它这么写, 第二次再来, 我觉得就没问题)

在这里插入图片描述

可重用 meta 节点实现

在这里插入图片描述

#include "common.h"
#include "index_handle.h"
#include "sstream"

static int debug = 1;

namespace qiniu
{
    namespace largefile
    {
        /* IndexHandle(const std::string &base_path, const uint32_t main_block_id); */
        IndexHandle::IndexHandle(const std::string &base_path, const uint32_t main_block_id)
        {
            /* create file_op_ handle object */
            std::stringstream tmp_stream;
            tmp_stream << base_path << INDEX_DIR_PREFIX << main_block_id; /* /root/weifc/index/ */

            std::string index_path;
            tmp_stream >> index_path;

            file_op_ = new MMapFileOperation(index_path, O_CREAT | O_RDWR | O_LARGEFILE);
            is_load_ = false;
        }


        IndexHandle::~IndexHandle()
        {
            if (file_op_)
            {
                delete file_op_;
                file_op_ = NULL;
            }
        }


        int IndexHandle::create(const uint32_t logic_block_id, const int32_t bucket_size, const MMapOption map_optiont)
        {
            int ret = TFS_SUCCESS;

            if (debug)
            {
                printf("create index, block id: %u, bucket size: %d, max_mmap_size: %d, first mmap size: %d, per mmap size: %d\n",
                        logic_block_id, bucket_size, map_optiont.max_mmap_size_, map_optiont.first_mmap_size_, map_optiont.per_mmap_size_);
            }

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();

            if (file_size < 0)
            {
                return TFS_ERROR;
            }else if(file_size == 0) /* empty file */
            {
                IndexHeader i_header;
                i_header.block_info_.block_id_ = logic_block_id;
                i_header.block_info_.seq_no_ = 1;
                i_header.bucket_size_ = bucket_size;

                i_header.index_file_size_ = sizeof(IndexHandle) + bucket_size * sizeof(int32_t);

                /* index header + total buckets */
                char *init_data = new char[i_header.index_file_size_];
                memcpy(init_data, &i_header, sizeof(IndexHandle));
                memset(init_data + sizeof(IndexHandle), 0, i_header.index_file_size_ - sizeof(IndexHandle));

                /* write index header and buckets into file */
                ret = file_op_->pwrite_file(init_data, i_header.index_file_size_, 0);

                delete []init_data;
                init_data = NULL;

                if (ret != TFS_ERROR)
                {
                    return ret;
                }

                ret = file_op_->flush_file();
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }
            }else /* file size > 0, index already exist */
            {
                return EXIT_META_UNEXPECT_FOUND_ERROR;
            }
            
            ret = file_op_->mmap_file(map_optiont);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }
            
            is_load_ = true;

            if (debug)
            {
                printf("init blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }


        int IndexHandle::load(const uint32_t logic_block_id, const int32_t _bucket_size, const MMapOption map_option)
        {
            int ret = TFS_SUCCESS;

            if (is_load_)
            {
                return EXIT_INDEX_ALREADY_LOADED_ERROR;
            }

            int64_t file_size = file_op_->get_file_size();
            if (file_size < 0)
            {
                return file_size;
            }
            else if (file_size == 0) /* empty file */
            {
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            MMapOption tmp_map_option = map_option;

            if (file_size > tmp_map_option.first_mmap_size_ && file_size <= tmp_map_option.max_mmap_size_)
            {
                tmp_map_option.first_mmap_size_ = file_size;
            }

            ret = file_op_->mmap_file(tmp_map_option);

            if (ret != TFS_SUCCESS)
            {
                return ret;
            }

            if (debug)
            {
                printf("InexHandle::load - bucket size(): %d, index_header()->bucket_size_: %d, block id: %d \n", bucket_size(), index_header()->bucket_size_, block_info()->block_id_);
            }

            if (0 == bucket_size() || 0 == block_info()->block_id_)
            {
                fprintf(stderr, "lndex corrupt error. blockid: %u, bucket size: %d\n", block_info()->block_id_, bucket_size());
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            /* check file size */
            int32_t index_file_size = sizeof(IndexHandle) + bucket_size() * sizeof(int32_t);

            if (file_size < index_file_size)
            {
                fprintf(stderr, "Index corrupt error. blockid: %u, buck size: %d, file size: %ld, index file size: %d\n", block_info()->block_id_, bucket_size(), file_size, index_file_size);
                return EXIT_INDEX_CORRUPT_ERROR;
            }

            /* check block_id_ size */
            if (logic_block_id != block_info()->block_id_)
            {
                fprintf(stderr, "block id conflict. blockid: %u, index blockid: %u\n", logic_block_id, block_info()->block_id_);
                return EXIT_BLOCKID_CONFLICT_ERROR;
            }

            /* check bucket_size */
            if (_bucket_size != bucket_size())
            {
                fprintf(stderr, "Index configure error, old bucket size: %d, new bucket size: %d\n", bucket_size(), _bucket_size);
                return EXIT_BUCKET_CONFIGURE_ERROR;
            }

            is_load_ = true;

            if (debug)
            {
                printf("load blockid: %d, index successful.data file size: %d, index file size: %d, bucket_size: %d, free head offset: %d, seqno: %d, size: %d, filecount: %d, del_size: %d, del_file_count: %d, version: %d\n", 
                logic_block_id, index_header()->data_file_offset_, index_header()->index_file_size_,
                index_header()->bucket_size_, index_header()->free_head_offset_, block_info()->seq_no_, block_info()->size_,
                block_info()->file_count_, block_info()->del_size_, block_info()->del_file_count_, block_info()->version_);
            }

            return TFS_SUCCESS;
        }


        int IndexHandle::remove(const uint32_t logic_block_id)
        {
            if (is_load_)
            {
                if (logic_block_id != block_info()->block_id_)
                {
                    fprintf(stderr, "block id conflict bolckid: %d, index biokid: %d\n", logic_block_id, block_info()->block_id_);
                    return EXIT_BLOCKID_CONFLICT_ERROR;
                }
            }
            
            int ret = file_op_->munmap_file();
            if (TFS_SUCCESS != ret)
            {
                return ret;
            }

            ret = file_op_->unlink_file();
            return ret;
        }


        int IndexHandle::flush()
        {
            int ret = file_op_->flush_file();

            if (TFS_SUCCESS != ret)
            {
                fprintf(stderr, "index flush fail, ret: %d error desc: %s\n", ret, strerror(errno));
            }

            return ret;
        }


         int IndexHandle::write_segment_meta(const uint64_t key, Metalnfo &meta)
         {
            int32_t current_offset = 0, previous_offset = 0;

            /* 思考? key 存在吗? 存在 => 处理? 不存在 => 处理? */
            /* 1. 从文件哈希表中查找 key 是否存在. hash_find(key, current_offset, previous_offset */
            int ret = hash_find(key, current_offset, previous_offset);
            if (TFS_SUCCESS == ret)
            {
                return EXIT_META_UNEXPECT_FOUND_ERROR;
            }else if (EXIT_META_NOT_FOUND_ERROR == ret)
            {
                return ret;
            }

            /* 2. 不存在就写入 meta 到文件哈希表中 hash_insert(key, previous_offset, meta) */
            ret = hash_insert(key, previous_offset, meta);
            return ret;

         }


         int32_t IndexHandle::read_segment_meta(const uint64_t key, Metalnfo &meta)
         {
            int32_t current_offset = 0, previous_offset = 0;

            /* 1. 确定 key 存放的桶(slot)的位置 */
            //int32_t slot = static_cast<uint32_t>(key) % bucket_size();

            int32_t ret = hash_find(key, current_offset, previous_offset);

            if (ret == TFS_SUCCESS) /* exist */
            {
                ret = file_op_->pread_file(reinterpret_cast<char *>(&meta), sizeof(Metalnfo), current_offset);
                return ret;
            }
            else
            {
                return ret;
            }
            
         }


         int32_t IndexHandle::delete_segment_meta(const uint64_t key) /* 删除 */
         {
            int32_t current_offset = 0, previous_offset = 0;

            int32_t ret = hash_find(key, current_offset, previous_offset);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }

            Metalnfo meta_info;

            ret = file_op_->pread_file(reinterpret_cast<char *>(&meta_info), sizeof(Metalnfo), current_offset);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }

            int32_t next_pos = meta_info.get_next_meta_offset();

            if (previous_offset == 0)
            {
                int32_t slot = static_cast<uint32_t>(key) % bucket_size();
                bucket_slot()[slot] = next_pos;
            }
            else
            {
                Metalnfo pre_meta_info;
                ret = file_op_->pread_file(reinterpret_cast<char *>(&pre_meta_info), sizeof(Metalnfo), previous_offset);
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }

                pre_meta_info.set_next_meta_offset(next_pos);

                ret = file_op_->pwrite_file(reinterpret_cast<char *>(&pre_meta_info), sizeof(Metalnfo), previous_offset);
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }
            }

            /* 把删除节点加入可重用节点链表(下一个小节实现) */
            meta_info.set_next_meta_offset(free_head_offset()); /* index_header()->free_head_offset_ */
            ret = file_op_->pwrite_file(reinterpret_cast<char *>(&meta_info), sizeof(Metalnfo), current_offset);
            if (ret != TFS_SUCCESS)
            {
                return ret;
            }

            index_header()->free_head_offset_ = current_offset;
            if (debug) 
            {
                printf("delete_segment_meta - reuse metainfo current_offset: %d\n", current_offset);
            }

            update_block_info(C_OPER_DELETE, meta_info.get_size());
            return TFS_SUCCESS;
         }


         int IndexHandle::hash_find(const uint64_t key, int32_t &current_offset, int32_t &previous_offset)
         {
            int ret = TFS_SUCCESS;
            Metalnfo meta_info;

            current_offset = 0;
            previous_offset = 0;

            /* 1. 确定 key 存放的桶(slot)的位置 */
            int32_t slot = (int32_t)key % bucket_size();

            /* 2. 读取桶首节点存储的第一个节点的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR */
            /* 3. 根据偏移量读取存储的metainfo */
            /* 4. 与key进行比较, 相等则设置 current_offset 和 previous_offset 并返回 TFS_SUCCESS, 否则继续执行5 */
            /* 5. 从 metainfo 中取得下一个节点的在文件中的偏移量, 如果偏移量为零, 直接返回 EXIT_META_NOT_FOUND_ERROR,
            否则继续跳转到3循环执行 */
            int32_t pos = bucket_slot()[slot];

            for (; pos != 0; )
            {
                ret = file_op_->pread_file(reinterpret_cast<char *>(&meta_info), sizeof(Metalnfo), pos);
                if (TFS_SUCCESS != ret)
                {
                    return ret;
                }

                if (hash_compare(key, meta_info.get_key()))
                {
                    current_offset = pos;
                    return TFS_SUCCESS;
                }

                previous_offset = pos;
                pos = meta_info.get_next_meta_offset();
            }

            return EXIT_META_NOT_FOUND_ERROR;
        }


        int32_t IndexHandle::hash_insert(const uint64_t key, int32_t previous_offset, Metalnfo &meta)
        {
            Metalnfo tmp_meta_info;
            int ret = TFS_SUCCESS;
            int32_t current_offset = 0;

            /* 1. 确定 key 存放的桶(slot)的位置 */
            int32_t slot = static_cast<uint32_t>(key) % bucket_size();

            /* 2. 确定 meta 节点存储中文件中的偏移量 */
            if (free_head_offset() != 0)
            {
                ret = file_op_->pread_file(reinterpret_cast<char *>(&tmp_meta_info), sizeof(Metalnfo), free_head_offset());
                if (ret != TFS_SUCCESS)
                {
                    return ret;
                }

                current_offset = index_header()->free_head_offset_;
                if (debug) 
                {
                    printf("reuse metainfo current_offset: %d\n", current_offset);
                }
                index_header()->free_head_offset_ = tmp_meta_info.get_next_meta_offset();
            }
            else
            {
                current_offset = index_header()->index_file_size_;
                index_header()->index_file_size_ += sizeof(Metalnfo);
            }
            

            /* 3. 将 meta 节点写入索引文件中 */
            meta.set_next_meta_offset(0); /* 我怀疑我忘了实现 */

            ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&meta), sizeof(Metalnfo), current_offset);
            if (ret != TFS_SUCCESS)
            {
                index_header()->index_file_size_ -= sizeof(Metalnfo);
                return ret;
            }

            /* 4. 将 meta 节点插入到哈希链表中 */
            /* 当前一个节点已经存在 */
            if (0 != previous_offset)
            {
                ret = file_op_->pread_file(reinterpret_cast<char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
                if (TFS_SUCCESS != ret)
                {
                    index_header()->index_file_size_ -= sizeof(Metalnfo);
                    return ret;
                }

                tmp_meta_info.set_next_meta_offset(current_offset);
                ret = file_op_->pwrite_file(reinterpret_cast<const char *>(&tmp_meta_info), sizeof(Metalnfo), previous_offset);
                if (TFS_SUCCESS != ret)
                {
                    index_header()->index_file_size_ -= sizeof(Metalnfo);
                    return ret;
                }
            }
            else /* 不存在前一个节点的情况 */
            {
                bucket_slot()[slot] = current_offset;
            }
            
            return TFS_SUCCESS;
        }


        int IndexHandle::update_block_info(const OperType oper_type, const uint32_t modify_size) /* 更新块信息 */
        {
            if (block_info()->block_id_ == 0)
            {
                return EXIT_BLOCKID_ZERO_ERROR;
            }

            if (oper_type == C_OPER_INSERT)
            {
                ++block_info()->version_;
                ++block_info()->file_count_;
                ++block_info()->seq_no_;
                block_info()->size_ += modify_size;
            }
            else if (oper_type == C_OPER_DELETE)
            {
                ++block_info()->version_;
                --block_info()->file_count_;
                block_info()->size_ -= modify_size;
                ++block_info()->del_file_count_;
                block_info()->del_size_ += modify_size;
            }

            if (debug)
            {
                printf("updata block info. blockid: %u, version: %u, file count: %u, size: %u, del file count: %u, del size: %u, seq no: %u, oper type: %d\n", 
                block_info()->block_id_, block_info()->version_, block_info()->file_count_, block_info()->size_,
                block_info()->del_file_count_, block_info()->del_size_, block_info()->seq_no_, oper_type);
            }

            return TFS_SUCCESS;
        }
    }
}


编译:

g++ block_delete_test.cpp file_op.cpp index_handle.cpp mmap_file.cpp  mmap_file_op.cpp -o block_delete_test

执行:

(我人傻了, 可能手抖或走神写错代码, 完全测试失败! 别问我错误原因, 我不知道, 因为这代码是别人的思想, 不过没有关系, 第一轮我也是有目的的-- 就是看到思想, 和为啥它这么写, 第二轮复习再来, 我觉得就没问题)
在这里插入图片描述

结束 - 文件系统核心存储引擎

结语:

总之: 第一轮了解它的思想, 编程的核心就是思想 - > 再实现

最后上传老师的代码如下:

https://download.csdn.net/download/m0_45867846/12564372

时间: 2020-06-30

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值