TFS文件系统策略分析

TFS文件系统策略分析

   首先我们来看一下TFS分布式文件系统的特点介绍:

   1.完全扁平化的数据组织结构，抛弃了传统文件系统的目录结构。

2.在块设备基础上建立自有的文件系统，减少EXT3等文件系统数据碎片带来的性能损耗

3.单进程管理单块磁盘的方式，摒除RAID5机制

4.带有HA机制的中央控制节点，在安全稳定和性能复杂度之间取得平衡。

5.尽量缩减元数据大小，将元数据全部加载入内存，提升访问速度。

6.跨机架和IDC的负载均衡和冗余安全策略。

7.完全平滑扩容

“源码之前,了无秘密”,了解TFS的最好方法是阅读源代码,这一组TFS文件系统的代码解析文章一共五篇,摘取的代码进行过筛选,基本上是在能体现TFS技术含量的或者是阅读代码比较有帮助的.

 这里选择两个场景,能够反映TFS在这两个场景里面分别采用的策略.

 (1). 处理来自数据服务器发送的心跳信息, 每次都需要判断是否要给该数据服务器创建新的逻辑块:

/*处理来自数据服务器发送的心跳信息*/ int HeartManagement::join_ds(Message* msg) {         SetDataserverMessage* message = dynamic_cast<SetDataserverMessage*> (msg);         //获得数据服务器状态         DataServerStatInfo *ds_stat_info = message->get_ds();         //server_id是数据服务器地址以及端口转换而来的         uint64_t server_id = ds_stat_info->id_;          RespHeartMessage *result_msg = new RespHeartMessage();         //如果数据服务器死亡         if (ds_stat_info->status_ == DATASERVER_STATUS_DEAD)         {             //释放该数据服务器             meta_mgr_.leave_ds(server_id);             result_msg->set_status(HEART_MESSAGE_OK);             message->reply_message(result_msg);             message->free();             return TFS_SUCCESS;         }         MetaManager::EXPIRE_BLOCK_LIST expire_list;         bool isnew = false;         //MetaManager& meta_mgr_ 加入数据服务器的处理方法         meta_mgr_.join_ds(*ds_stat_info, isnew);         //如果isnew==TRUE 表示新加入的数据服务器         if (isnew)         {             TBSYS_LOG(INFO, "dataserver(%s) join: use capacity(%" PRI64_PREFIX "u),total capacity(%" PRI64_PREFIX "u), has_block(%s)",                 tbsys::CNetUtil::addrToString(server_id).c_str(), ds_stat_info->use_capacity_,                 ds_stat_info->total_capacity_, message->get_has_block() == HAS_BLOCK_FLAG_YES ? "Yes" : "No");             if (meta_mgr_.get_fs_name_system()->get_ns_global_info()->owner_role_ == NS_ROLE_MASTER)             {                 replicate_lancher_.inc_stop_balance_count();             }         }         //如果有逻辑块的汇报信息         if (message->get_has_block() == HAS_BLOCK_FLAG_YES)         {             //MetaManager& meta_mgr_ 处理发送来的报告该数据服务器上的数据块信息             meta_mgr_.report_blocks(server_id, *message->get_blocks(), expire_list);             //如果命名服务器的角色是主命名服务器             if (meta_mgr_.get_fs_name_system()->get_ns_global_info()->owner_role_ == NS_ROLE_MASTER)             {                 uint32_t expire_blocks_size = 0;                 uint32_t i = 0;                 MetaManager::EXPIRE_BLOCK_LIST::iterator iter = expire_list.begin();                 for (; iter != expire_list.end(); ++iter)                 {                     if (iter->first == server_id)                     {                         vector < uint32_t > &expire_blocks = iter->second;                         expire_blocks_size = expire_blocks.size();                         for (i = 0; i < expire_blocks_size; ++i)                         {                             if (!replicate_lancher_.get_executor().is_replicating_block(expire_blocks[i]))                                 result_msg->add_expire_id(expire_blocks[i]);                         }                     }                     else                     {                         //对失效的数据块,除了自己本身,在删除其他服务器上的块                         NameServer::rm_block_from_ds(iter->first, iter->second);                     }                 }             }             result_msg->set_status(HEART_EXP_BLOCK_ID);             TBSYS_LOG(INFO, "dataserver(%s) join: use capacity(%" PRI64_PREFIX "u),total capacity(%" PRI64_PREFIX "u), block count(%u)",                 tbsys::CNetUtil::addrToString(server_id).c_str(), ds_stat_info->use_capacity_,                 ds_stat_info->total_capacity_, message->get_blocks()->size());         }         else         {             //如果该数据服务器汇报来的心跳信息不包含有逻辑块信息             ServerCollect* servre_collect = meta_mgr_.get_block_ds_mgr().get_ds_collect(server_id);             int32_t block_count = -1;             if (servre_collect != NULL){                 block_count = servre_collect->get_block_list().size();             }             //如果是新加的数据服务器,并且逻辑块列表数位,那么返回告诉需要汇报逻辑块信息             if (isnew || servre_collect->get_block_list().size() == 0)             {                 TBSYS_LOG(INFO, "reply dataserver(%s) heart msg need send block, isnew(%s),current block count(%u)",                     tbsys::CNetUtil::addrToString(server_id).c_str(), isnew ? "true" : "false", block_count);                 result_msg->set_status(HEART_NEED_SEND_BLOCK_INFO);             }             else             {                 result_msg->set_status(HEART_MESSAGE_OK);             }             if ((meta_mgr_.get_fs_name_system()->get_ns_global_info()->switch_time_ != 0) && (block_count                 != ds_stat_info->block_count_))             {                 TBSYS_LOG(DEBUG, "new ds block count(%d): old ds block count(%d)", ds_stat_info->block_count_, block_count);                   if (time(NULL) < (meta_mgr_.get_fs_name_system()->get_ns_global_info()->switch_time_                     + SYSPARAM_NAMESERVER.ds_dead_time_))                     result_msg->set_status(HEART_NEED_SEND_BLOCK_INFO);                 else                     meta_mgr_.get_fs_name_system()->get_ns_global_info()->switch_time_ = 0;             }           }         message->reply_message(result_msg);         if (message->get_has_block() == HAS_BLOCK_FLAG_YES){             //SYSPARAM_NAMESERVER.add_primary_block_count_=5:一次添加可写块的数量             //每次都需要判断是否要给该数据服务器创建新的逻辑块             meta_mgr_.check_primary_writable_block(server_id, SYSPARAM_NAMESERVER.add_primary_block_count_);         }         message->free();         message = NULL;         return TFS_SUCCESS; }

 

  meta_mgr_.check_primary_writable_block()方法:

//该数据服务器需要添加的主逻辑块,如果小于设定的能做主数据服务器的逻辑块数,发送创建新的逻辑块的消息     //add_block_count:一次添加可写块的数量=add_primary_block_count = 5 一次添加可写块的数量 int MetaManager::check_primary_writable_block(const uint64_t ds_id, const int32_t add_block_count, bool promote)     {         ServerCollect* server_collect = meta_mgr_.get_ds_collect(ds_id);         int32_t need_add_block_count = 0;         if (server_collect != NULL)         {             //该数据服务器磁盘空间满,直接返回             if (server_collect->is_disk_full()){                 return 0;             }             //得到当前可以该数据服务器可以提供的作为主逻辑块的列表的个数             int32_t current = static_cast<int32_t> (server_collect->get_primary_writable_block_list()->size());             //参数:max_write_filecount = 5 dataserver在一段时间拥有可写块的最大数             if (current >= SYSPARAM_NAMESERVER.max_write_file_count_)             {                 TBSYS_LOG(INFO, "check primary writableblock in dataserver(%s), current_primary_block_count(%u) >= max_write_file_count(%d), no need to add new block",                     tbsys::CNetUtil::addrToString(ds_id).c_str(), current, SYSPARAM_NAMESERVER.max_write_file_count_);                 return 0;             }             //获得需要添加的逻辑块数             need_add_block_count = std::min(add_block_count, (SYSPARAM_NAMESERVER.max_write_file_count_ - current));             TBSYS_LOG(INFO, "check primary writableblock in dataserver(%s), current primary block count(%u), need add block count(%d)",                 tbsys::CNetUtil::addrToString(ds_id).c_str(), current, need_add_block_count);         }         if (need_add_block_count > 0)         {             if (promote){                 promote_primary_write_block(server_collect, need_add_block_count);             }             //如果命名服务器不是主控服务器,那么直接返回             if (fs_name_system_->get_ns_global_info()->owner_role_ != NS_ROLE_MASTER){                 return 0;             }             //