创建Glusterfs分布式RAID10卷

Glusterfs3.2.4/5支持五种Volume，即Distribute卷、Stripe卷、Replica卷、Distribute stripe卷和Distribute replica卷，这五种卷可以满足不同应用对高性能、高可用的需求。
(1) distribute volume：分布式卷，文件通过hash算法分布到brick server上，这种卷是glusterfs的基础和最大特点；
(2) stripe volume：条带卷，类似RAID0，条带数=brick server数量，文件分成数据块以Round Robin方式分布到brick server上，并发粒度是数据块，大文件性能高；
(3) replica volume：镜像卷，类似RAID1，镜像数=brick server数量，所以brick server上文件数据相同，构成n-way镜像，可用性高；
(4) distribute stripe volume：分布式条带卷，brick server数量是条带数的倍数，兼具distribute和stripe卷的特点；
(5)d istribute replica volume：分布式镜像卷，brick server数量是镜像数的倍数，兼具distribute和replica卷的特点；

Stripe volume和replica volume分别相当于RAID0和RAID1，前者可以获得更高的并发性和性能，后者可以获得更高的可用性。如果能够同时获得高性能和高可用性，类似RAID10/RAID01，应该是更加完美。Glusterfs采用堆栈式技术架构，基于distribute, stripe, replica卷可以采用堆积木的方式构建复合卷，因此构建RAID10/RAID01是完全可能的。可惜的是，目前Glusterfs3.2.4/3.2.5管理工具gluster还不支持创建stripe replica volume和distribute stripe replica volume。最近发布的3.2.5管理手册说支持，但管理工具没有同步支持。3.3beta管理工具支持，但是使用起来有问题，多次创建均没有成功。

实际上，我们可以通过修改glusterfs卷配置文件来实现分布式RAID10，即distribute stripe replica volume。Stripe replica volume是一种特殊分布式RAID10卷，即stripe count * replica count = birck server数量。这种方式绕过Glusterd/mgmt来管理，能够实现功能，但是修改起来不方便，建议谨慎使用。简单操作步骤如下：
(1)使用gluster工具创建distribute stripe卷；
(2)停止所有相关brick server上的glusterd服务，/etc/init.d/glusterd stop；
(3)在一个brick server上手工修改相应的卷配置信息，然后同步至所有相关brick server；
(4)重启所有相关brick server上的glusterd服务，/etc/init.d/glusterd start。
下面我们举个实际的例子，这里为了简化在一台服务器上操作。

(1)创建distribute stripe卷

gluster volume create raid10 stripe 2 192.168.75.129:/opt/raid10-1 192.168.75.129:/opt/raid10-2 192.168.75.129:/opt/raid10-3 192.168.75.129:/opt/raid10-4 
 192.168.75.129:/opt/raid10-5 192.168.75.129:/opt/raid10-6 192.168.75.129:/opt/raid10-7 192.168.75.129:/opt/raid10-8

得到的/etc/glusterd/vols/raid10/raid10-fuse.vol配置文件内容如下：

volume raid10-client-0
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-1
    option transport-type tcp
end-volume

volume raid10-client-1
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-2
    option transport-type tcp
end-volume

volume raid10-client-2
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-3
    option transport-type tcp
end-volume

volume raid10-client-3
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-4
    option transport-type tcp
end-volume

volume raid10-client-4
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-5
    option transport-type tcp
end-volume

volume raid10-client-5
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-6
    option transport-type tcp
end-volume

volume raid10-client-6
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-7
    option transport-type tcp
end-volume

volume raid10-client-7
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-8
    option transport-type tcp
end-volume

volume raid10-stripe-0
    type cluster/stripe
    subvolumes raid10-client-0 raid10-client-1
end-volume

volume raid10-stripe-1
    type cluster/stripe
    subvolumes raid10-client-2 raid10-client-3
end-volume

volume raid10-stripe-2
    type cluster/stripe
    subvolumes raid10-client-4 raid10-client-5
end-volume

volume raid10-stripe-3
    type cluster/stripe
    subvolumes raid10-client-6 raid10-client-7
end-volume

volume raid10-dht
    type cluster/distribute
    subvolumes raid10-stripe-0 raid10-stripe-1 raid10-stripe-2 raid10-stripe-3
end-volume

volume raid10-write-behind
    type performance/write-behind
    subvolumes raid10-dht
end-volume

volume raid10-read-ahead
    type performance/read-ahead
    subvolumes raid10-write-behind
end-volume

volume raid10-io-cache
    type performance/io-cache
    subvolumes raid10-read-ahead
end-volume

volume raid10-quick-read
    type performance/quick-read
    subvolumes raid10-io-cache
end-volume

volume raid10-stat-prefetch
    type performance/stat-prefetch
    subvolumes raid10-quick-read
end-volume

volume raid10
    type debug/io-stats
    option latency-measurement off
    option count-fop-hits off
    subvolumes raid10-stat-prefetch
end-volume

(2) service glusterd stop

(3)编辑/etc/glusterd/vols/raid10/raid10-fuse.vol，修改后内容如下：

volume raid10-client-0
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-1
    option transport-type tcp
end-volume

volume raid10-client-1
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-2
    option transport-type tcp
end-volume

volume raid10-client-2
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-3
    option transport-type tcp
end-volume

volume raid10-client-3
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-4
    option transport-type tcp
end-volume

volume raid10-client-4
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-5
    option transport-type tcp
end-volume

volume raid10-client-5
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-6
    option transport-type tcp
end-volume

volume raid10-client-6
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-7
    option transport-type tcp
end-volume

volume raid10-client-7
    type protocol/client
    option remote-host 192.168.75.129
    option remote-subvolume /opt/raid10-8
    option transport-type tcp
end-volume

volume raid10-afr-0
    type cluster/replicate
    subvolumes raid10-client-0 raid10-client-1
end-volume

volume raid10-afr-1
    type cluster/replicate
    subvolumes raid10-client-2 raid10-client-3
end-volume

volume raid10-afr-2
    type cluster/replicate
    subvolumes raid10-client-4 raid10-client-5
end-volume

volume raid10-afr-3
    type cluster/replicate
    subvolumes raid10-client-6 raid10-client-7
end-volume

volume raid10-stripe-0
    type cluster/stripe
    subvolumes raid10-afr-0 raid10-afr-1
end-volume

volume raid10-stripe-1
    type cluster/stripe
    subvolumes raid10-afr-2 raid10-afr-3
end-volume

volume raid10-dht
    type cluster/distribute
    subvolumes raid10-stripe-0 raid10-stripe-1
end-volume

volume raid10-write-behind
    type performance/write-behind
    subvolumes raid10-dht
end-volume

volume raid10-read-ahead
    type performance/read-ahead
    subvolumes raid10-write-behind
end-volume

volume raid10-io-cache
    type performance/io-cache
    subvolumes raid10-read-ahead
end-volume

volume raid10-quick-read
    type performance/quick-read
    subvolumes raid10-io-cache
end-volume

volume raid10-stat-prefetch
    type performance/stat-prefetch
    subvolumes raid10-quick-read
end-volume

volume raid10
    type debug/io-stats
    option latency-measurement off
    option count-fop-hits off
    subvolumes raid10-stat-prefetch
end-volume

(4) service glusterd start; gluster volume start raid10
(5) gluster volume info

Volume Name: raid10
Type: Distributed-Stripe
Status: Started
Number of Bricks: 4 x 2 = 8
Transport-type: tcp
Bricks:
Brick1: 192.168.75.129:/opt/raid10-1
Brick2: 192.168.75.129:/opt/raid10-2
Brick3: 192.168.75.129:/opt/raid10-3
Brick4: 192.168.75.129:/opt/raid10-4
Brick5: 192.168.75.129:/opt/raid10-5
Brick6: 192.168.75.129:/opt/raid10-6
Brick7: 192.168.75.129:/opt/raid10-7

至此，Glusterfs分布式RAID10卷已经创建好的，开始测试和应用吧!