swift存储策略实践

 

1   一个装好swift源码的环境
      安装的方式，见上一篇博文
      环境ip请根据自己的需要设置。
2   存储策略应用
    将一个安装好的集群环境，通过指定存储策略，可以实现同一个节点使用多块硬盘。


通过该文档，可以实现的应用方面：
    1、针对不同类型的数据，指定不同的container使用不同的副本数。
    2、针对不同的性能需求，指定不同的container使用不同存储设备。
    3、针对不同的地域，根据container划分存储路径。


3   Switf单节点支持多硬盘
3.1  第一块硬盘准备(在每一个安装swift的节点)
    #这个是使用dd模拟，相应的，只要将dd方式改成硬盘分区的方式，即可。
    #为了方便查询，分开准备
    #其实所有的操作命令都是一致的，可以抽象成一个方法
    #其余创建xfs格式方式：http://www.jb51.net/LINUXjishu/248321.html
    (创建swift用户：group add swift;useradd -g swift swift)


    mkdir -p /srv
    dd if=/dev/zero of=/srv/swift-disk1 bs=1024 count=0 seek=1000000
    mkfs.xfs -i size=1024 /srv/swift-disk1
        #Edit /etc/fstab and add
    /srv/swift-disk1 /mnt/part1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
    mkdir /mnt/part1
    mount /mnt/part1
    mkdir /mnt/part1/1
    ln -s /mnt/part1/1 /srv/1
    chown -R swift:swift /etc/swift /srv/1/ /var/run/swift /mnt/part1


3.2  第二块盘准备
    mkdir -p /srv
    dd if=/dev/zero of=/srv/swift-disk2 bs=1024 count=0 seek=1000000
    mkfs.xfs -i size=1024 /srv/swift-disk2
        #Edit /etc/fstab and add


    #Edit /etc/fstab and add
    /srv/swift-disk2 /mnt/part2 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
    mkdir /mnt/part2
    mount /mnt/part2
    mkdir /mnt/part2/1
    ln -s /mnt/part2/1 /srv/2
    chown -R swift:swift /etc/swift /srv/2/ /var/run/swift  /mnt/part2


3.3   第三块盘准备
    mkdir -p  /srv
    dd if=/dev/zero of=/srv/swift-disk3 bs=1024 count=0 seek=1000000
    mkfs.xfs -i size=1024 /srv/swift-disk3
        #Edit /etc/fstab and add
    /srv/swift-disk3 /mnt/part3 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
    mkdir /mnt/part3
    mount /mnt/part3
    mkdir /mnt/part3/1
    ln -s /mnt/part3/1 /srv/3


    chown -R swift:swift /etc/swift /srv/3/ /var/run/swift  /mnt/part3


3.4   第四块盘准备
    mkdir -p /srv
    dd if=/dev/zero of=/srv/swift-disk4 bs=1024 count=0 seek=1000000
    mkfs.xfs -i size=1024 /srv/swift-disk4
        #Edit /etc/fstab and add
    /srv/swift-disk4 /mnt/part4 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
    mkdir /mnt/part4
    mount /mnt/part4
    mkdir /mnt/part4/1
    ln -s /mnt/part4/1 /srv/4


    chown -R swift:swift /etc/swift /srv/4/ /var/run/swift /mnt/part4




4  配置文件设置
    rsyncd.conf的配置根根据每增加一个设备路径(一个挂载点)，就增加一个相应账户/容器/对象的挂载路径的配置。


4.1   /etc/rsyncd.conf
    secrets file = /etc/rsyncd.secrets
    uid = root
    gid = root
    use chroot = no
    max connections = 5
    log file = /var/log/rsyncd.log
    pid file = /var/run/rsyncd.pid
    lock file = /var/run/rsync.lock


    [domain]
    comment = welcome
    path = /var/www/domain/
    auth users = root
    hosts allow = 127.0.0.1
    hosts deny = *
    list = true
    read only = no




    [account6002] 
    max connections = 25 
    path = /srv/1/node/ 
    read only = false 
    lock file = /var/lock/account6002.lock 
    [account6005] 
    max connections = 25 
    path = /srv/2/node/ 
    read only = false 
    lock file = /var/lock/account6005.lock 
    [account6008] 
    max connections = 25 
    path = /srv/3/node/ 
    read only = false 
    lock file = /var/lock/account6008.lock 




    [container6001] 
    max connections = 25 
    path = /srv/1/node/ 
    read only = false 
    lock file = /var/lock/container6001.lock 
    [container6004] 
    max connections = 25 
    path = /srv/2/node/ 
    read only = false 
    lock file = /var/lock/container6004.lock 
    [container6007] 
    max connections = 25 
    path = /srv/3/node/ 
    read only = false 
    lock file = /var/lock/container6007.lock 


    [object6000] 
    max connections = 25 
    path = /srv/1/node/ 
    read only = false 
    lock file = /var/lock/object6000.lock 
    [object6003] 
    max connections = 25 
    path = /srv/2/node/ 
    read only = false 
    lock file = /var/lock/object6003.lock 
    [object6006] 
    max connections = 25 
    path = /srv/3/node/ 
    read only = false 
    lock file = /var/lock/object6006.lock


    最后执行：
    rsync –daemon
    如果出现rsync已经运行的情况，需要kill掉进程以及删除其记录进程pid的文件。


4.2  /etc/swift/swift.conf
    [swift-hash]
    # random unique strings that can never change (DO NOT LOSE)
    swift_hash_path_suffix = 1ac4bfb54486ee0f


    [storage-policy:0]
    name = apple
    default = yes


    [storage-policy:1]
    name = ssd
    default = no


4.2.1  注意点
      这个文件配置的就是swift的存储策略，[storage-policy:0]是默认的存储策略(default=yes)，建议使用该存储策略为默认的存储策略，因为swift的代码有限制。
     [storage-policy:1]为第二个存储策略。
     存储策略不能随意增加，在构建object ring的时候，代码会校验/etc/swift的路径下是否有对应的 object*.ring.gz的文件。
      需要注意的[storage-policy:0]使用的建环命令是：
    swift-ring-builder object.builder create 18 3 1
    swift-ring-builder object.builder add z1-192.168.2.202:6000/sdb1 1
    而[storage-policy:1]使用的建环命令是：
    swift-ring-builder object-1.builder create 18 3 1
    swift-ring-builder object-1.builder add z3-192.168.2.214:6000/part1 1


    之所以有这样的问题，是由于代码写死了策略0使用的 ring数据位 object.ring.gz。就是这点需要特别注意下。
4.3  /etc/swift/proxy-server.conf
    [DEFAULT]
    bind_port = 8080
    workers = 8
    user = swift
    log_facility = LOG_LOCAL1


    [pipeline:main]
    pipeline = healthcheck cache proxy-logging proxy-server
    # pipeline = healthcheck cache proxy-logging proxy-server


    [app:proxy-server]
    use = egg:swift#proxy
    allow_account_management = true
    account_autocreate = true
    log_facility = LOG_LOCAL1
    log_level = DEBUG


    [filter:cache]
    use = egg:swift#memcache
    memcache_servers = 127.0.0.1:11211  


    [filter:catch_errors]
    use = egg:swift#catch_errors


    [filter:healthcheck]
    use = egg:swift#healthcheck


    [filter:proxy-logging]
    use = egg:swift#proxy_logging


    [filter:keystone]
    use = egg:swift#keystoneauth
    operator_roles = admin, SwiftOperator
    is_admin = true
    cache = swift.cache


    [filter:authtoken]
    paste.filter_factory = keystonemiddleware.auth_token:filter_factory
    admin_tenant_name = %SERVICE_TENANT_NAME%
    admin_user = %SERVICE_USER%
    admin_password = %SERVICE_PASSWORD%
    auth_host = 127.0.0.1
    auth_port = 35357
    auth_protocol = http
    signing_dir = /tmp/keystone-signing-swift


4.4  account-server
4.4.1.1  注意点
    由于当前主要验证account-server也可以支持配置在单节点多硬盘的环境中，正常环境应该是每个account-server在一个节点上就使用其中的一个硬盘，然后分别配置在3个节点上，保证数据的可靠性。
    同理，container-server也建议每个节点只配置一份，然后配置在3个节点上。
同一个节点中，每一个account-server/container-server/object-server的端口号不能冲突。建议account-server的端口号取值为 6002,6005,6008,6011,6014,6017,6020,6023，container-server的端口号为6001,6004,6007,6010,6013,6016,6019,6022，object-server的端口号为6000,6003,6006,6009,6012,6015,6018,6021,6024,6025,6026,6027…….。object-server的端口号可以预留多一点，account-server以及container-server的端口号可以留的少一点。
端口号的数量是跟同一个节点中，挂载路径的数量来决定的。一般account-server/container-server只使用一个挂载路径，object-server根据硬盘的多少，可能会使用多个挂载路径，所以object-server可能预留的要稍微多点，但一般10-20个盘已经够多 了。


4.4.2  /etc/swift/account-server/1.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    bind_port = 6002
    workers = 2
    mount_check = false
    log_facility = LOG_LOCAL1
    devices = /srv/1/node


    [pipeline:main]
    pipeline = account-server


    [app:account-server]
    use = egg:swift#account


    [account-replicator]
    log_name = account-replicator
    log_facility = LOG_LOCAL1


    [account-auditor]
    log_name = account-auditor
    log_facility = LOG_LOCAL1


    [account-reaper]
    log_facility = LOG_LOCAL1


4.4.3  /etc/swift/account-server/2.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    bind_port = 6005
    workers = 2
    mount_check = false
    log_facility = LOG_LOCAL1
    devices = /srv/2/node


    [pipeline:main]
    pipeline = account-server


    [app:account-server]
    use = egg:swift#account


    [account-replicator]
    log_name = account-replicator
    log_facility = LOG_LOCAL1


    [account-auditor]
    log_name = account-auditor
    log_facility = LOG_LOCAL1


    [account-reaper]
    log_facility = LOG_LOCAL1






4.4.4  /etc/swift/account-server/3.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    bind_port = 6008
    workers = 2
    mount_check = false
    log_facility = LOG_LOCAL1
    devices = /srv/3/node


    [pipeline:main]
    pipeline = account-server


    [app:account-server]
    use = egg:swift#account


    [account-replicator]
    log_name = account-replicator
    log_facility = LOG_LOCAL1


    [account-auditor]
    log_name = account-auditor
    log_facility = LOG_LOCAL1


    [account-reaper]
    log_facility = LOG_LOCAL1






4.5 container-server
4.5.1 /etc/swift/container-server/1.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6001
    log_facility = LOG_LOCAL1
    devices = /srv/1/node


    [pipeline:main]
    pipeline = container-server


    [app:container-server]
    use = egg:swift#container


    [container-replicator]


    [container-updater]


    [container-auditor]


    [container-sync]


4.5.2 /etc/swift/container-server/2.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6004
    log_facility = LOG_LOCAL1
    devices = /srv/2/node


    [pipeline:main]
    pipeline = container-server


    [app:container-server]
    use = egg:swift#container


    [container-replicator]


    [container-updater]


    [container-auditor]


    [container-sync]




4.5.3 /etc/swift/container-server/3.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6007
    log_facility = LOG_LOCAL1
    devices = /srv/3/node


    [pipeline:main]
    pipeline = container-server


    [app:container-server]
    use = egg:swift#container


    [container-replicator]


    [container-updater]


    [container-auditor]


    [container-sync]




4.6 object-server
4.6.1  /etc/swift/object-server/1.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6000
    log_facility = LOG_LOCAL1
    devices = /srv/1/node
    [pipeline:main]
    pipeline = object-server
    [app:object-server]
    use = egg:swift#object
    [object-replicator]
    [object-updater]
    [object-auditor]


4.6.2  /etc/swift/object-server/2.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6000
    log_facility = LOG_LOCAL1
    devices = /srv/2/node
    [pipeline:main]
    pipeline = object-server
    [app:object-server]
    use = egg:swift#object
    [object-replicator]
    [object-updater]
    [object-auditor]


4.6.3  /etc/swift/object-server/3.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6000
    log_facility = LOG_LOCAL1
    devices = /srv/3/node
    [pipeline:main]
    pipeline = object-server
    [app:object-server]
    use = egg:swift#object
    [object-replicator]
    [object-updater]
    [object-auditor]


4.6.4  /etc/swift/object-server/4.conf
    [DEFAULT]
    bind_ip = 192.168.2.214
    workers = 2
    mount_check = false
    bind_port = 6009
    log_facility = LOG_LOCAL1
    devices = /srv/4/node
    [pipeline:main]
    pipeline = object-server
    [app:object-server]
    use = egg:swift#object
    [object-replicator]
    [object-updater]
    [object-auditor]




4.7  拷贝文件到指定目录
    文件修改后


    删除/etc/swift下面的无效文件


    只保留：


    ├── account-server
    │   ├── 1.conf
    │   ├── 2.conf
    │   └── 3.conf
    ├── container-server
    │   ├── 1.conf
    │   ├── 2.conf
    │   └── 3.conf
    │   ├── 1.conf
    │   ├── 2.conf
    │   ├── 3.conf
    │   └── 4.conf
    ├── proxy-server.conf
    └── swift.conf


4.8  创建ring
4.8.1  注意点
    构建换建议到/etc/swift路径下进行操作，在其他路径操作的话，需要再将数据拷贝到/etc/swift路径下，拷贝完后，需要将/etc/swift下面的权限修改成 swift:swift，否则swift进程无法启动。


    swift-ring-builder account.builder create 18 3 1
    18表示 hash环的partion 2**18个
    (这里的2^18个为swift中一致性hash中涉及到的虚拟机节点的个数，而非环节点个数(swift中的partion即为虚拟节点)，环节点数固定为2^32。（MD5的hash算法的值范围）);


    swift-ring-builder account.builder add z1-192.168.2.202:6002/sdb1 1
    这个命令添加设备，端口号会跟我们account-server.conf中的配置的端口号相同的配置文件中，指定的挂载路径进行数据的存储，1表示权重。
    其中每个节点需要使用z开头，否则代码不认。


   所有ring创建好后，可以将这些数据拷贝到所有节点上，所有节点共用同一份配置数据。
   其中*.ring.gz的生成是在执行swift-ring-builder account.builder rebalance后生成的。
4.8.2  命令
    cd /etc/swift
    swift-ring-builder account.builder create 18 3 1
    swift-ring-builder account.builder add z1-192.168.2.202:6002/sdb1 1
    swift-ring-builder account.builder add z2-192.168.2.203:6002/sdb1 1
    swift-ring-builder account.builder add z3-192.168.2.214:6002/part1 1
    swift-ring-builder account.builder add z3-192.168.2.214:6005/part2 1
    swift-ring-builder account.builder add z3-192.168.2.214:6008/part3 1
    swift-ring-builder account.builder
    swift-ring-builder account.builder rebalance


    swift-ring-builder container.builder create 18 3 1
    swift-ring-builder container.builder add z1-192.168.2.202:6001/sdb1 1
    swift-ring-builder container.builder add z2-192.168.2.203:6001/sdb1 1
    swift-ring-builder container.builder add z3-192.168.2.214:6001/part1 1
    swift-ring-builder container.builder add z3-192.168.2.214:6004/part2 1
    swift-ring-builder container.builder add z3-192.168.2.214:6007/part3 1


    swift-ring-builder container.builder
    swift-ring-builder container.builder rebalance


    swift-ring-builder object.builder create 18 3 1
    swift-ring-builder object.builder add z1-192.168.2.202:6000/sdb1 1
    swift-ring-builder object.builder add z2-192.168.2.203:6000/sdb1 1
    swift-ring-builder object.builder add z3-192.168.2.214:6009/part4 1
    swift-ring-builder object-1.builder create 18 3 1
    swift-ring-builder object-1.builder add z3-192.168.2.214:6000/part1 1
    swift-ring-builder object-1.builder add z3-192.168.2.214:6003/part2 1
    swift-ring-builder object-1.builder add z3-192.168.2.214:6006/part3 1


    swift-ring-builder object.builder
    swift-ring-builder object-1.builder
    swift-ring-builder object.builder rebalance
    swift-ring-builder object-1.builder rebalance


    groupadd swift
    useradd -g swift swift
    chown -R swift:swift /etc/swift/


4.9   创建swift用户
    keystone user-create --name swift --pass swift
    keystone user-role-add --user swift --tenant services --role admin
    keystone service-create --name swift --type object-store --description "OpenStack Object Storage"
    keystone endpoint-create \
      --service-id $(keystone service-list | awk '/ object-store / {print $2}') \
      --publicurl 'http://havip:8080/v1/AUTH_%(tenant_id)s' \
      --internalurl 'http://havip:8080/v1/AUTH_%(tenant_id)s' \
      --adminurl http://havip:8080 \
      --region regionOne


    配置iptable


    iptables -I INPUT -p tcp -m multiport --dports 6000 -m comment --comment "swift" -j ACCEPT
    


4.10    测试
4.10.1 创建account
publicURL=http://192.168.2.214:8080/v1/AUTH_test
curl -i $publicURL
curl -i $publicURL  -X POST


4.10.2 使用默认的container:
curl -i $publicURL/apple  -X PUT -H "Content-Length: 0"


4.10.3  创建制定存储策略的container
curl -i $publicURL/ssd  -X PUT -H  "X-Storage-Policy: ssd"
curl -k -i -X PUT -T "swift.conf" $publicURL/ssd/swift.conf




查看可以看到，有多个副本，同时都在 同一个存储策略指定的分区中：


swift-get-nodes /etc/swift/object-1.ring.gz AUTH_test ssd swift.conf  
 


当objects-1指定的全是同一个型号的硬盘时候，即可实现性能分层。
当objects-1全指定特定的主机的时候，即可实现按域存储。

 

 

当构建object-1的ring的时候，指定的副本数根据需要设定的时候，还可以实现不同container的根据需求，存储不同的副本数。

 

 

由于环境各异，也可能自己写的有所遗漏，可能安装后出现问题，欢迎大家相互交流，一起学习。