ceph存储 fuse_operations与fuse_lowlevel_ops

fuse为开发者提供了两组接口，分别是fuse_lowlevel_ops以及fuse_operations，开发者只需要实现这两组接口的一种即可实现一个用户空间文件系统。

 

struct fuse_lowlevel_ops的成员如下所示，其中init方法在其它所有方法之前调用，用于初始化文件系统，fuse已经实现，destroy则是在文件系统被卸载时做一些清理工作。用于大多数请求的参数都是fuse_ino_t类型的ino，而文件系统提供给用户的视图是以文件名呈现的，故lookup是实现文件系统的关键，它在parent中查找名字name对应的文件，并返回相应的信息，可使用fuse_reply_entry或fuse_reply_err作为请求的返回。

 

接口中的方法对于了解过VFS的人应该都不难理解，只要按需实现这些接口，你就可以定制出属于自己的文件系统，这组接口的详细说明见fuse_lowlevel.h。

 

void(*	init )(void *userdata, struct fuse_conn_info *conn)
void(*	destroy )(void *userdata)
void(*	lookup )(fuse_req_t req, fuse_ino_t parent, const char *name)
void(*	forget )(fuse_req_t req, fuse_ino_t ino, unsigned long nlookup)
void(*	getattr )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
void(*	setattr )(fuse_req_t req, fuse_ino_t ino, struct stat *attr, int to_set, struct fuse_file_info *fi)
void(*	readlink )(fuse_req_t req, fuse_ino_t ino)
void(*	mknod )(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode, dev_t rdev)
void(*	mkdir )(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode)
void(*	unlink )(fuse_req_t req, fuse_ino_t parent, const char *name)
void(*	rmdir )(fuse_req_t req, fuse_ino_t parent, const char *name)
void(*	symlink )(fuse_req_t req, const char *link, fuse_ino_t parent, const char *name)
void(*	rename )(fuse_req_t req, fuse_ino_t parent, const char *name, fuse_ino_t newparent, const char *newname)
void(*	link )(fuse_req_t req, fuse_ino_t ino, fuse_ino_t newparent, const char *newname)
void(*	open )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
void(*	read )(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info *fi)
void(*	write )(fuse_req_t req, fuse_ino_t ino, const char *buf, size_t size, off_t off, struct fuse_file_info *fi)
void(*	flush )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
void(*	release )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
void(*	fsync )(fuse_req_t req, fuse_ino_t ino, int datasync, struct fuse_file_info *fi)
void(*	opendir )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
void(*	readdir )(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info *fi)
void(*	releasedir )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
void(*	fsyncdir )(fuse_req_t req, fuse_ino_t ino, int datasync, struct fuse_file_info *fi)
void(*	statfs )(fuse_req_t req, fuse_ino_t ino)
void(*	setxattr )(fuse_req_t req, fuse_ino_t ino, const char *name, const char *value, size_t size, int flags)
void(*	getxattr )(fuse_req_t req, fuse_ino_t ino, const char *name, size_t size)
void(*	listxattr )(fuse_req_t req, fuse_ino_t ino, size_t size)
void(*	removexattr )(fuse_req_t req, fuse_ino_t ino, const char *name)
void(*	access )(fuse_req_t req, fuse_ino_t ino, int mask)
void(*	create )(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode, struct fuse_file_info *fi)
void(*	getlk )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi, struct flock *lock)
void(*	setlk )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi, struct flock *lock, int sleep)
void(*	bmap )(fuse_req_t req, fuse_ino_t ino, size_t blocksize, uint64_t idx)
void(*	ioctl )(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, struct fuse_file_info *fi, unsigned *flagsp, const void *in_buf, size_t in_bufsz, size_t out_bufszp)
void(*	poll )(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi, struct fuse_pollhandle *ph)

 

用户实现的接口是如何跟这个结构关联起来的？

其实fuse中已经实现了一组接口，在fuse_lowlevel.c中，定义了一个静态的结构数组，该数组的元素为一组（函数，名字）的结构，但没做什么实际的工作，当fuse用户空间的daemon从/fuse/dev中读取到请求之后，它通过请求号来判别各个请求，并调用这里相应的处理函数，如读取到read调用时，会调用do_read进行处理。

 

static struct {

    void (*func)(fuse_req_t, fuse_ino_t, const void *);

    const char *name;

} fuse_ll_ops[] = {

    //只列举了部分

    [FUSE_LOOKUP]      = { do_lookup,      "LOOKUP"      },

    [FUSE_OPEN]        = { do_open,        "OPEN"        },

    [FUSE_READ]        = { do_read,        "READ"        },

    [FUSE_WRITE]       = { do_write,       "WRITE"       },

    [FUSE_STATFS]      = { do_statfs,      "STATFS"      },

    [FUSE_FLUSH]       = { do_flush,       "FLUSH"       },

    [FUSE_INIT]        = { do_init,        "INIT"        },

    [FUSE_OPENDIR]     = { do_opendir,     "OPENDIR"     },

    [FUSE_READDIR]     = { do_readdir,     "READDIR"     },

    [FUSE_RELEASEDIR]  = { do_releasedir,  "RELEASEDIR"  },

    [FUSE_DESTROY]     = { do_destroy,     "DESTROY"     }

};

 

 

接下来看一下do_read的实现

static void do_read(fuse_req_t req, fuse_ino_t nodeid, const void *inarg)

{

    struct fuse_read_in *arg = (struct fuse_read_in *) inarg;

    // 如果用户实现了read操作，则调用用户空间的read，否则以没有实现该调用为错误响应，这里的op就是用户实现文件系统时实现的，并传递给fuse。

    if (req->f->op.read) {

        struct fuse_file_info fi;

        memset(&fi, 0, sizeof(fi));

        fi.fh = arg->fh;

        fi.fh_old = fi.fh;

        req->f->op.read(req, nodeid, arg->size, arg->offset, &fi);

    } else

        fuse_reply_err(req, ENOSYS);

}

 

从这里的实现可以看出，这些操作是没有加任何锁的，如果开发者需要文件系统锁，需要在实现文件系统时自行考虑。

 

fuse_operations又是怎么一回事？

对于实现fuse_lowlevel_ops这组接口，没有内核VFS相关知识的开发者是不可能完成的，为了增强fuse的通用性，使更多的用户能够使用fuse开发文件系统，fuse提供了一组更简单的接口fuse_operations，详细说明请参考fuse.h。这组接口的参数跟unix提供的系统调用的参数很类似，开发者更易理解，fuse想开发者屏蔽了底层的相关对象，直接以文件名作为参数，只有开发者按照自己的方式，把这组接口实现就可以，显然这比上面那组接口的实现要简单得多。

int(*	getattr )(const char *, struct stat *)
int(*	readlink )(const char *, char *, size_t)
int(*	mknod )(const char *, mode_t, dev_t)
int(*	mkdir )(const char *, mode_t)
int(*	unlink )(const char *)
int(*	rmdir )(const char *)
int(*	symlink )(const char *, const char *)
int(*	rename )(const char *, const char *)
int(*	link )(const char *, const char *)
int(*	chmod )(const char *, mode_t)
int(*	chown )(const char *, uid_t, gid_t)
int(*	truncate )(const char *, off_t)
int(*	utime )(const char *, struct utimbuf *)
int(*	open )(const char *, struct fuse_file_info *)
int(*	read )(const char *, char *, size_t, off_t, struct fuse_file_info *)
int(*	write )(const char *, const char *, size_t, off_t, struct fuse_file_info *)
int(*	statfs )(const char *, struct statvfs *)
int(*	flush )(const char *, struct fuse_file_info *)
int(*	release )(const char *, struct fuse_file_info *)
int(*	fsync )(const char *, int, struct fuse_file_info *)
int(*	setxattr )(const char *, const char *, const char *, size_t, int)
int(*	getxattr )(const char *, const char *, char *, size_t)
int(*	listxattr )(const char *, char *, size_t)
int(*	removexattr )(const char *, const char *)
int(*	opendir )(const char *, struct fuse_file_info *)
int(*	readdir )(const char *, void *, fuse_fill_dir_t, off_t, struct fuse_file_info *)
int(*	releasedir )(const char *, struct fuse_file_info *)
int(*	fsyncdir )(const char *, int, struct fuse_file_info *)
void *(*	init )(struct fuse_conn_info *conn)
void(*	destroy )(void *)
int(*	access )(const char *, int)
int(*	create )(const char *, mode_t, struct fuse_file_info *)
int(*	ftruncate )(const char *, off_t, struct fuse_file_info *)
int(*	fgetattr )(const char *, struct stat *, struct fuse_file_info *)
int(*	lock )(const char *, struct fuse_file_info *, int cmd, struct flock *)
int(*	utimens )(const char *, const struct timespec tv[2])
int(*	bmap )(const char *, size_t blocksize, uint64_t *idx)
unsigned int	flag_nullpath_ok: 1
unsigned int	flag_reserved: 31
int(*	ioctl )(const char *, int cmd, void *arg, struct fuse_file_info *, unsigned int flags, void *data)
int(*	poll )(const char *, struct fuse_file_info *, struct fuse_pollhandle *ph, unsigned *reventsp)

  

提供这组接口，fuse做了什么？

fuse还是实现了一组fuse_lowlevel_ops的接口，在fuse.c中

static struct fuse_lowlevel_ops fuse_path_ops = {

    //只列举了部分方法

    .init = fuse_lib_init,

    .destroy = fuse_lib_destroy,

    .lookup = fuse_lib_lookup,

    .forget = fuse_lib_forget,

    .getattr = fuse_lib_getattr,

    .setattr = fuse_lib_setattr,

.access = fuse_lib_access,

.read = fuse_lib_read,

    .readlink = fuse_lib_readlink

};

 

fuse实现的这组接口跟之前的方法不一样，不是什么都不做，它完成了部分工作，主要是文件节点与文件名的转换关系，然后将文件名作为参数，调用用户实现的fuse_operations的接口。

 

如fuse_lib_read的实现

int fuse_fs_read(struct fuse_fs *fs, const char *path, char *buf, size_t size,

                 off_t off, struct fuse_file_info *fi)

{

fuse_get_context()->private_data = fs->user_data;

//用户实现的方法

    if (fs->op.read)

        return fs->op.read(path, buf, size, off, fi);

    else

        return -ENOSYS;

}

 

static void fuse_lib_read(fuse_req_t req, fuse_ino_t ino, size_t size,

                          off_t off, struct fuse_file_info *fi)

{

    struct fuse *f = req_fuse_prepare(req);

    char *path;

    char *buf;

    int res;

 

    buf = (char *) malloc(size);

    if (buf == NULL) {

        reply_err(req, -ENOMEM);

        return;

    }

 

    res = -ENOENT;

pthread_rwlock_rdlock(&f->tree_lock); //fuse_operations使用了读写锁

//由ino获取path

    path = get_path(f, ino);

    if (path != NULL) {

        struct fuse_intr_data d;

        if (f->conf.debug)

            fprintf(stderr, "READ[%llu] %lu bytes from %llu\n",

                    (unsigned long long) fi->fh, (unsigned long) size,

                    (unsigned long long) off);

 

        fuse_prepare_interrupt(f, req, &d);

        res = fuse_fs_read(f->fs, path, buf, size, off, fi); //通过这个方法调用用户实现的方法

        fuse_finish_interrupt(f, req, &d);

        free(path);

    }

    pthread_rwlock_unlock(&f->tree_lock);

 

    if (res >= 0) {

        if (f->conf.debug)

            fprintf(stderr, "   READ[%llu] %u bytes\n",

                    (unsigned long long)fi->fh, res);

        if ((size_t) res > size)

            fprintf(stderr, "fuse: read too many bytes");

        fuse_reply_buf(req, buf, res); //返回结果

    } else

        reply_err(req, res);

 

    free(buf);

 

从上面的代码可以看出，fuse对fuse_operations这组操作使用的是读写锁，而不是互斥量，这样有利于提升文件系统执行效率。当读写锁是写加锁状态时，在它被解锁之前，所有试图对这个锁加锁的线程都会被阻塞。当读写锁在读加锁状态时，所有试图以读模式对它进行加锁的线程都可以获得访问权，但是如果线程希望以写模式对此锁加锁，它必须阻塞直到所有的线程释放读锁。在不同的系统上读写锁的实现可能各不相同，但当读写锁处于读模式锁住状态时，如果有另外线程试图以写模式加锁时，读写锁通常会阻塞接下来的读模式锁请求，以避免读模式锁被长期占用，导致写模式锁请求很长时间不能被满足。