一、rbd内核驱动写入流程
1)初始化
首先是rbd驱动的初始化工作:包括验证libceph的兼容性,分配内存,在sysfs中创建块设备控制文件、创建工作队列rbd_wq并调用INIT_WORK初始化它
module_init(rbd_init);
static int __init rbd_init(void)
{
if (!libceph_compatible(NULL)) { //兼容性
rbd_warn(NULL, "libceph incompatibility (quitting)");
return -EINVAL;
}
rbd_wq = alloc_workqueue(RBD_DRV_NAME, WQ_MEM_RECLAIM, 0); //创建工作队列
if (!rbd_wq)
{
rc = -ENOMEM;
goto err_out_slab;
}
rc = rbd_slab_init(); //初始化内存分配器
if (rc)
return rc;
.......
rc = rbd_sysfs_init(); //创建/sys/bus/rbd/
if (rc)
goto err_out_blkdev;
...}
static int rbd_init_request(void *data, struct request *rq,
unsigned int hctx_idx, unsigned int request_idx,
unsigned int numa_node)
{
struct work_struct *work = blk_mq_rq_to_pdu(rq);
INIT_WORK(work, rbd_queue_workfn); //初始化一个work,work通过rbd_queue_workfn进行处理
return 0;
}
2)块设备创建、工作队列中启动work
添加块设备,首先创建一个rbd client用来通信,然后选择一个pool存储池去创建rbd设备,创建完成后调用rbd_dev_device_setup初始化rbd设备,在初始化块设备的时候会启动工作队列rbd_wq,并将通用块设备层的请求转化为一个work添加到rbd_wq工作队列中,然后由cpu调度执行工作队列rbd_wq中的work,work对应的处理函数为rbd_queue_workfn,该work用于处理通用块设备层的IO请求。
启动work的调用关系: rbd_dev_device_setup → rbd_init_disk → rbd_mq_ops → rbd_init_request → rbd_queue_workfn处理函数
static BUS_ATTR(add, S_IWUSR, NULL, rbd_add);
static BUS_ATTR(remove, S_IWUSR, NULL, rbd_remove);
static BUS_ATTR(add_single_major, S_IWUSR, NULL, rbd_add_single_major);
static BUS_ATTR(remove_single_major, S_IWUSR, NULL, rbd_remove_single_major);
static int rbd_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
{
struct request *rq = bd->rq;
struct work_struct *work = blk_mq_rq_to_pdu(rq); //通用块设备层请求转为work
queue_work(rbd_wq, work); //将work加入到工作队列,工作队列中的work由cpu调度处理
return BLK_MQ_RQ_QUEUE_OK;
}
static ssize_t rbd_add(struct bus_type *bus,
const char *buf,
size_t count)
{
if (single_major)
return -EINVAL;
return do_rbd_add(bus, buf, count);
}
static ssize_t do_rbd_add(struct bus_type *bus,
const char *buf,
size_t count)
{
.....
rbdc = rbd_get_client(ceph_opts); //获取或创建rbd_client
if (IS_ERR(rbdc)) {
rc = PTR_ERR(rbdc);
goto err_out_args;
}
/* pick the pool */
rc = rbd_add_get_pool_id(rbdc, spec->pool_name); //选择存储池
if (rc < 0) {
if (rc == -ENOENT)
pr_info("pool %s does not exist\n", spec->pool_name);
goto err_out_client;
}
spec->pool_id = (u64)rc;
rbd_dev = rbd_dev_create(rbdc, spec, rbd_opts); //创建rbd设备
down_write(&rbd_dev->header_rwsem);
......
rc = rbd_dev_image_probe(rbd_dev, 0); //探针更多的是检查rbd image是否被map
if (rc < 0) {
up_write(&rbd_dev->header_rwsem);
goto err_out_rbd_dev;
}
......
rc = rbd_dev_device_setup(rbd_dev); //包括obj->pg映射等
static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
{
int ret;
......
/* Set up the blkdev mapping. */
ret = rbd_init_disk(rbd_dev);
......
}
static int rbd_init_disk(struct rbd_device *rbd_dev)
{
struct gendisk *disk;
struct request_queue *q;
u64 segment_size;
int err;
.....
memset(&rbd_dev->tag_set, 0, sizeof(rbd_dev->tag_set));
rbd_dev->tag_set.ops = &rbd_mq_ops; //rbd_dev初始化
rbd_dev->tag_set.queue_depth = rbd_dev->opts->queue_depth;
rbd_dev->tag_set.numa_node = NUMA_NO_NODE;
.....
}
static struct blk_mq_ops rbd_mq_ops = {
.queue_rq = rbd_queue_rq,
.init_request = rbd_init_request, //调用rbd_init_request
};
static int rbd_init_request(void *data, struct request *rq,
unsigned int hctx_idx, unsigned int request_idx,
unsigned int numa_node)
{
struct work_struct *work = blk_mq_rq_to_pdu(rq);
INIT_WORK(work, rbd_queue_workfn); //通过work_struct启动线程
return 0;
}
3)work处理函数rbd_queue_workfn内流程分析
从上层取出通用块设备层请求后,转换为image对象,再从image对象批量转为object对象,再计算出object到pg,pg到osd的映射关系。
3.1 获取通用块设备层信息
在rbd_queue_workfn中,通过blk_mq_rq_from_pdu获取到通用块设备层IO请求rq、通过blk_rq_bytes(rq)获取到请求中需要写入的数据长度length(length表示的是客户端需要写到磁盘总的数据长度),通过blk_rq_pos(rq)获取块设备写入偏移量offset。
static void rbd_queue_workfn(struct work_struct *work)
{
struct request *rq = blk_mq_rq_from_pdu(work); //通用块设备层请求
struct rbd_device *rbd_dev = rq->q->queuedata;
struct rbd_img_request *img_request;
struct ceph_snap_context *snapc = NULL;
u64 offset = (u64)blk_rq_pos(rq) << SECTOR_SHIFT; //块设备的偏移量
u64 length = blk_rq_bytes(rq); //
enum obj_operation_type op_type;
.....
}
3.2 通用块设备层信息转换image请求,image请求批量转换为object
在rbd_queue_workfn中从通用块设备层请求中获取到块设备偏移offset和长度length后,再使用这些指标来创建img_request并将img_request→offset进行填充中,然后调用rbd_img_request_fill函数,在该函数中,基于rados object的大小(4M)与rados对象在rbd中的segment排列,对请求进行拆分,最终将rbd_img_request拆分成多个rbd_obj_request对象,通过这样的过程实现从linux内核的通用块请求到ceph rados object的转换。
static void rbd_queue_workfn(struct work_struct *work)
{
struct request *rq = blk_mq_rq_from_pdu(work);
struct rbd_device *rbd_dev = rq->q->queuedata;
struct rbd_img_request *img_request;
u64 offset = (u64)blk_rq_pos(rq) << SECTOR_SHIFT; //块设备偏移
u64 length = blk_rq_bytes(rq); //长度
......
img_request = rbd_img_request_create(rbd_dev, offset, length, op_type, //创建img_request
snapc);
img_request->offset = offset; //填充img_request→offset
result = rbd_img_request_fill(img_request, OBJ_REQUEST_BIO, //将rbd_img_request划分为一个个rbd_obj_request
rq->bio);
.....
}
static int rbd_img_request_fill(struct rbd_img_request *img_request,
enum obj_request_type type,
void *data_desc)
{
struct rbd_obj_request *obj_request = NULL;
u64 img_offset;
img_offset = img_request->offset; //块设备当前写入的偏移位置
resid = img_request->length; //待写入的长度
while (resid) {
......
object_name = rbd_segment_name(rbd_dev, img_offset); //对象名
length = rbd_segment_length(rbd_dev, img_offset, resid); //长度
obj_request = rbd_obj_request_create(object_name, //创建obj_request对象
offset, length, type);
......
img_offset += length; //偏移增加length
resid -= length;
......
}
3.3 rbd块设备offset到rados object的映射
rbd块设备到rados对象的映射是根据rados对象的大小以及当前块设备的偏移量来决定的,并且rados对象的命名方式采用前缀rbd_data.$image_id.16位16进制的序号构成。
3.3.1 rados对象大小与命名方式
每个rbd块设备都定义了一个2为底的指数来表示每个rbd对象的大小,这个指数称为rbd的obj order。obj order默认值为22,因此每个rbd对象大小2^22Bytes,即每个rados对象大小为4MB。注意:这个是默认值,是可以通过rbd命令进行修改的。