移植facebook flashcache模块支持 linux kernel 4.13以上

为什么80%的码农都做不了架构师？>>>   

环境

准备在ubuntu 18.04 lts下搭建kvm+libvirt+glusterfs的虚拟化环境。

4台服务器是2块SSD，4块SATA（5400RPM）。

尝试了gluster tier，效果不好。

对于虚拟化环境下大文件的存储，tier的promote和demote的策略很难设置，

在大文件发生大量数据交换时会出现tier层"no space left"的错误，文件无法写入。

在没有更好的解决方案之前，facebook的flashcache（https://github.com/facebookarchive/flashcache )对于提速SATA盘的作用还是非常显著的。

但ubuntu18.04的apt源还没有。

https://launchpad.net/flashcache/+packages

错误

于是下载flashcache源码编译dkms模块的时候，报错：

DKMS make.log for flashcache-3.1.3+git20150701 for kernel 4.15.0-36-generic (x86_64)
Fri Oct 12 15:06:00 UTC 2018
make -C /lib/modules/4.15.0-36-generic/build M=/var/lib/dkms/flashcache/3.1.3+git20150701/build modules V=0
make[1]: Entering directory '/usr/src/linux-headers-4.15.0-36-generic'
Makefile:975: "Cannot use CONFIG_STACK_VALIDATION=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel"
  CC [M]  /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_conf.o
  CC [M]  /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_main.o
  CC [M]  /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o
/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.c: In function ‘flashcache_bio_endio’:
/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.c:742:7: error: ‘struct bio’ has no member named ‘bi_error’; did you mean ‘bi_iter’?
  bio->bi_error = error;
       ^~~~~~~~
       bi_iter
scripts/Makefile.build:332: recipe for target '/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o' failed
make[2]: *** [/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o] Error 1
Makefile:1551: recipe for target '_module_/var/lib/dkms/flashcache/3.1.3+git20150701/build' failed
make[1]: *** [_module_/var/lib/dkms/flashcache/3.1.3+git20150701/build] Error 2
make[1]: Leaving directory '/usr/src/linux-headers-4.15.0-36-generic'
Makefile:35: recipe for target 'modules' failed
make: *** [modules] Error 2

OS内核的版本是4.15.0-36。

原因

根据错误提示：bio->bierror ‘struct bio’ has no member named ‘bierror’;

找了4.15的内核源码看看bio的struct的定义,

bio是在include/linux/blktypes.h中定义。

https://elixir.bootlin.com/linux/v4.15/source/include/linux/blktypes.h

struct bio {
    struct bio      *bi_next;   /* request queue link */
    struct gendisk      *bi_disk;
    unsigned int        bi_opf;     /* bottom bits req flags,
                         * top bits REQ_OP. Use
                         * accessors.
                         */
    unsigned short      bi_flags;   /* status, etc and bvec pool number */
    unsigned short      bi_ioprio;
    unsigned short      bi_write_hint;
    blk_status_t        bi_status;
    u8          bi_partno;

    /* Number of segments in this BIO after
     * physical address coalescing is performed.
     */
    unsigned int        bi_phys_segments;

    /*
     * To keep track of the max segment size, we account for the
     * sizes of the first and last mergeable segments in this bio.
     */
    unsigned int        bi_seg_front_size;
    unsigned int        bi_seg_back_size;

    struct bvec_iter    bi_iter;

    atomic_t        __bi_remaining;
    bio_end_io_t        *bi_end_io;

    void            *bi_private;
#ifdef CONFIG_BLK_CGROUP
    /*
     * Optional ioc and css associated with this bio.  Put on bio
     * release.  Read comment on top of bio_associate_current().
     */
    struct io_context   *bi_ioc;
    struct cgroup_subsys_state *bi_css;
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
    void            *bi_cg_private;
    struct blk_issue_stat   bi_issue_stat;
#endif
#endif
    union {
#if defined(CONFIG_BLK_DEV_INTEGRITY)
        struct bio_integrity_payload *bi_integrity; /* data integrity */
#endif
    };

    unsigned short      bi_vcnt;    /* how many bio_vec's */

    /*
     * Everything starting with bi_max_vecs will be preserved by bio_reset()
     */

    unsigned short      bi_max_vecs;    /* max bvl_vecs we can hold */

    atomic_t        __bi_cnt;   /* pin count */

    struct bio_vec      *bi_io_vec; /* the actual vec list */

    struct bio_set      *bi_pool;

    /*
     * We can inline a number of vecs at the end of the bio, to avoid
     * double allocations for a small number of bio_vecs. This member
     * MUST obviously be kept at the very end of the bio.
     */
    struct bio_vec      bi_inline_vecs[0];
};

里面根本没有bierror。 再翻翻以前版本的代码，在4.12的bio定义里发现了bierror： https://elixir.bootlin.com/linux/v4.12.14/source/include/linux/blk_types.h

应该是从4.13版本内核之后，bio的结构定义发生了变化。

struct bio {
    struct bio      *bi_next;   /* request queue link */
    struct block_device *bi_bdev;
    int         bi_error;
    unsigned int        bi_opf;     /* bottom bits req flags,
                         * top bits REQ_OP. Use
                         * accessors.
                         */
    unsigned short      bi_flags;   /* status, etc and bvec pool number */
    unsigned short      bi_ioprio;

解决

修改flashcache_subr.c，将 bi_error改成bi_status。

重新编译，ok。

 

更新：

flashcache_subr.c按如下修改，可以兼容以前版本的内核。

void
flashcache_bio_endio(struct bio *bio, int error,
                     struct cache_c *dmc, struct timeval *start_time)
{
        if (unlikely(dmc->sysctl_io_latency_hist &&
                     start_time != NULL &&
                     start_time->tv_sec != 0))
                flashcache_record_latency(dmc, start_time);
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
        bio_endio(bio, bio->bi_size, error);
#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0)
        bio_endio(bio, error);
#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,13,0)
        bio->bi_error = error;
        bio_endio(bio);
#else
        bio->bi_status = error;
        bio_endio(bio);
#endif
}
 

 

转载于:https://my.oschina.net/hand/blog/2244957