CEPH CRUSH 的简单的模拟检测

通过代码分析crush从一个字符串到特定的存储设备的流程。 #简单的测试代码如下：

// g++ hash_test.cc -fpermissive -Wcpp -lpthread
// VERSION v0.2
// PATH ./ceph/src/crush


#include <iostream>
#include <stdint.h>
#include <string.h>
#include <vector>
#include <stdlib.h>
#include <stdio.h>

#include "include/blobhash.h"
#include "include/ceph_fs.h"
#include "hash.h"
#include "../osd/osd_types.h"

#include "CrushWrapper.h"
#include "mapper.c"
#include "crush.c"
#include "builder.c"
#include "../config.cc"

using namespace std;



int main()
{
    string oid("noe-obj");    //创建一个noe-obj的对象
    const char *p1=oid.data();
    blobhash blo;
    size_t st;
    //hash string
    st = blo(p1,oid.length()); //将noe-obj使用size_t强转,生成size_t的一串数字(include/blobhash.h)
    cout<<st<<"\n";
    printf("123333%d\n",11);
    //hash format seed
    uint32_t hash32;
    hash32 = rjhash32(st);    //将size_t的一串数字做hash运算，产生一个32位的hash值
    cout<<hash32<<"\n";
    cout<<hex<<hash32<<"\n"; //将hash值转为16进制，在文件中体现在“751139FF”
                             //(/var/lib/ceph/osd/ceph-10/current/9.ff_head/rbd\\udata.b8b5279002d9.0000000000000335__head_751139FF__9)

    //PGID
    pg_t pgid(1,2,hash32,0,-1);  //生成pg对象(osd/osd_types.h)，
                                 //pg_t(int type, int size, ps_t seed, int pool, int pref)
                                 //参数分别代表:
                                 //type:CEPH_PG_TYPE_REP是1,CEPH_PG_TYPE_RAID4是2; 
                                 //size:备份数;
                                 //seed:位置的种子，即hash产生的32位的值; 
                                 //pool:存储池的id,rbd的id为0;  
                                 //pref：主osd的id，没有为-1；

    //CRUSH MAP
    int c1;
    c1 = ceph_stable_mod(hash32,64,63); //将已得到的hash值分配到pg上，c1为pg
                                        //(/include/ceph_fs.h) int ceph_stable_mod(int x, int b, int bmask)
                                        //参数分别代表：
                                        //x:  已经得到的hash值
                                        //b: pgp 已分配的pg的数量
                                        // bmask: pgp-1   
    cout<<"crush mon:"<<c1<<"\n";

    //crush seed
    unsigned crush1;
    crush1 = crush_hash32_2(c1,0); //计算pg_id和pool_id的hash值，得到32位的crush输入值crush1,
                                   //(crush/hash.h) unsigned crush_hash32_2(unsigned a, unsigned b)
                                   //参数分别代表
                                   //a:PG_id;
                                   //b: pool_id;
    cout<<crush1<<"\n";
//do_rule  
    int len,ids[2],idw[2];
    string map_file = "/root/crushmap.file";
    string out_file = "/root/Tcrushmap.file";
    string type_name[11] = {"root","region","datacenter","room","pod","pdu","row","rack","chassis","host","osd"}; //创建bucket的type
    const char *map = map_file.data();
    vector<int> out;
    CrushWrapper cw1;
    //types
    //cw1.read_from_file(map);
    cw1.create();  //初始化crushmap
    //Set Max devices
    //cw1.set_max_devices(18);
    // Set types
    for (int i=0;i<(sizeof(type_name)/sizeof(type_name[0]));i++)
    {
        const char *type = type_name[i].data();
        cout<<i<<"   "<<type_name[i]<<std::endl;
        cw1.set_type_name(10-i,type);   //创建bucket的type
                                        //(crush/CrushWrapper.h) void set_type_name(int i, const char *n)
                                        //参数分别代表：
                                        // i: type的id，例如：0,1,2,3. 0代表devices
                                        //n : 代表type的name
    }

    int dids[1];
    int didw[1] = {3};
    //Add the devices
    for (int i=0;i<18;i++)
    {
        char osd[10];
        sprintf(osd,"osd.%d",i);
        cout<<osd<<std::endl;
        cw1.set_item_name(i,osd); //设置item的name
                                  //(crush/CrushWrapper.h) void set_item_name(int i, const char *n)
                                  //参数分别代表
                                  //i: 代表item的id
                                  //n:代表需要设置的item的name
    }

    //root type and weight
    ids[0] = -4;
    ids[1] = -6;
    idw[0] = 12;
    idw[1] = 16;
    cw1.add_bucket(-1,CRUSH_BUCKET_STRAW,10,2,ids,idw);// 创建bucket -1
                                                       //(crush/CrushWrapper.h) int add_bucket(int bucketno, int alg, int type, int size,int *items, int *weights)
                                                       //参数分别代表
                                                       // bucketno:代表bucket的id
                                                       //alg: bucket 的算法分别有
                                                       /*   enum {
                                                                        CRUSH_BUCKET_UNIFORM = 1,
                                                                        CRUSH_BUCKET_LIST = 2,
                                                                        CRUSH_BUCKET_TREE = 3,
                                                                        CRUSH_BUCKET_STRAW = 4 
                                                                      };
                                                        */
                                                       //type:代表bucket的type，前面已创建的type
                                                       //size：代表包含多少的items，即items的长度
                                                       //items：包含的关系
                                                       //weights：权重
                                                       //
    cw1.set_item_name(-1,"default");     //设置bucket -1 的name为default
    //host type and weight
    int hids[3] = {0,6,7};
    int hidw[3] = {1,1,1};
    cout<<hidw[0]<<std::endl;
    cw1.add_bucket(-2,CRUSH_BUCKET_STRAW,1,3,hids,hidw);//创建bucket -2，bucket的类型为host，
                                                                                                       //包含3个items
    cw1.set_item_name(-2,"dstor-285c_cache");    //设置bucket -2的name为dstor-285c_cache
    //root type and weight
    ids[0] = -2;
    ids[1] = -5;
    idw[0] = 3;
    idw[1] = 3;
    cw1.add_bucket(-3,CRUSH_BUCKET_STRAW,10,2,ids,idw);//创建bucket -3，其类型为root,name为ssd_cache
    cw1.set_item_name(-3,"ssd_cache");
    //host type and weight
    int host4_ids[5] = {1,2,3,4,5};
    int host4_idw[5] = {3,3,3,3,2};
    cw1.add_bucket(-4,CRUSH_BUCKET_STRAW,1,5,host4_ids,host4_idw);
    cw1.set_item_name(-4,"dstor-285c");

    //host type and weight
    int host5_ids[3] = {8,16,17};
    int host5_idw[3] = {1,1,1};
    cw1.add_bucket(-5,CRUSH_BUCKET_STRAW,1,3,host5_ids,host5_idw);
    cw1.set_item_name(-5,"dstor-ad98_cache");

    //host type and weight
    int host6_ids[7] = {9,10,11,12,13,14,15};
    int host6_idw[7] = {3,3,3,3,3,3,3};
    cw1.add_bucket(-6,CRUSH_BUCKET_STRAW,1,7,host6_ids,host6_idw);
    cout<<"*********"<<cw1.get_bucket_item_weight(-6,9)<<std::endl;
    cw1.set_item_name(-6,"dstor-ad98");

    //Add the rule
    cw1.add_rule(3,0,1,1,10,0); //创建rule
                                //(crush/CrushWrapper.h) int add_rule(int len, int pool, int type, int minsize, int maxsize, int ruleno)
                                //参数分别代表：
                                //len: step的个数
                                //pool: pool的id
                                //type: CEPH_PG_TYPE_REP是1,CEPH_PG_TYPE_RAID4是2;
                                //minsize:
                                //maxsize:
                                // ruleno: rule的id
                                //

    cw1.set_rule_name(0,"replicated_ruleset");//修改rule的name
                                              //(crush/CrushWrapper.h) void set_rule_name(int i, const char *n)
                                              //其中i: 代表 rule_id
                                              //n:代表需要修改的name
                                              //
    cw1.set_rule_step(0,0,CRUSH_RULE_TAKE,-1,0);//设置rule的step
                                                //(crush/CrushWrapper.h)  int set_rule_step(unsigned ruleno, unsigned step, int op, int arg1, int arg2)
                                                //其中 ruleno 代表 rule_id
                                                //step:代表steps中的位置
                                                //op：
                                                /* enum { 
                                                       CRUSH_RULE_NOOP = 0,      
                                                       CRUSH_RULE_TAKE = 1,       // arg1 = value to start with
                                                       CRUSH_RULE_CHOOSE_FIRSTN = 2, // arg1 = num items to pick
                                                                                                                        // arg2 = type               
                                                        CRUSH_RULE_CHOOSE_INDEP = 3,  // same 2 
                                                        CRUSH_RULE_EMIT = 4           // no args 
                                                         };*/
                                                //
    cw1.set_rule_step(0,1,CRUSH_RULE_CHOOSE_FIRSTN,0,1);
    cw1.set_rule_step(0,2,CRUSH_RULE_EMIT,0,0);

    //Set SSD    
    cw1.add_rule(3,0,1,1,10,1);//增加一个step的长度为3的id为1，操作为备份模式的rule
    cw1.set_rule_name(1,"ssd_ruleset");//修改rule 1 的name为ssd_ruleset
    cw1.set_rule_step(1,0,CRUSH_RULE_TAKE,-3,0);
    cw1.set_rule_step(1,1,CRUSH_RULE_CHOOSE_FIRSTN,0,1);
    cw1.set_rule_step(1,2,CRUSH_RULE_EMIT,0,0);

    //Output file
    const char *outMap = out_file.data();
    cout<<"Write file"<<std::endl;
    cw1.write_to_file(outMap);//将crushmap写入文件/root/Tcrushmap.file
    //cw1.finalize();
    cw1.do_rule(0, crush1, out, 2, -1);
    cout<<"Crush Out is :"<<out<<std::endl;

    return 0;
}

#创建的crushmap如下：

# begin crush map

# devices
device 0 osd.0 offload -6115.720
device 1 osd.1 offload 0.496
device 2 osd.2 offload -6115.720
device 3 osd.3 offload 0.496
device 4 osd.4 offload 0.000
device 5 osd.5
device 6 osd.6 offload 0.001
device 7 osd.7
device 8 osd.8 offload 4470.468
device 9 osd.9 offload 0.500
device 10 osd.10 offload 4470.468
device 11 osd.11 offload 0.500
device 12 osd.12
device 13 osd.13
device 14 osd.14 offload 0.001
device 15 osd.15
device 16 osd.16 offload 66.249
device 17 osd.17

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root

# buckets
root default {
	id -1		# do not change unnecessarily
	alg straw
	item dstor-285c weight 0.000
	item dstor-ad98 weight 0.000
}
host dstor-285c_cache {
	id -2		# do not change unnecessarily
	alg straw
	item osd.0 weight 0.000
	item osd.6 weight 0.000
	item osd.7 weight 0.000
}
root ssd_cache {
	id -3		# do not change unnecessarily
	alg straw
	item dstor-285c_cache weight 0.000
	item dstor-ad98_cache weight 0.000
}
host dstor-285c {
	id -4		# do not change unnecessarily
	alg straw
	item osd.1 weight 0.000
	item osd.2 weight 0.000
	item osd.3 weight 0.000
	item osd.4 weight 0.000
	item osd.5 weight 0.000
}
host dstor-ad98_cache {
	id -5		# do not change unnecessarily
	alg straw
	item osd.8 weight 0.000
	item osd.16 weight 0.000
	item osd.17 weight 0.000
}
host dstor-ad98 {
	id -6		# do not change unnecessarily
	alg straw
	item osd.9 weight 0.000
	item osd.10 weight 0.000
	item osd.11 weight 0.000
	item osd.12 weight 0.000
	item osd.13 weight 0.000
	item osd.14 weight 0.000
	item osd.15 weight 0.000
}

# rules
rule replicated_ruleset {
	pool 0
	type replicated
	min_size 1
	max_size 10
	step take default
	step choose firstn 0 type host
	step emit
}
rule ssd_ruleset {
	pool 0
	type replicated
	min_size 1
	max_size 10
	step take ssd_cache
	step choose firstn 0 type host
	step emit
}

# end crush map

转载于:https://my.oschina.net/u/2242006/blog/1162598