Heketi + GlusterFS 搭建测试环境

环境准备

节点信息

• 172.19.35.70
  • GlusterFS 节点，Heketi 节点
• 172.19.35.71
  • GlusterFS 节点
• 172.19.35.72
  • GlusterFS 节点

初始化

每台机器上执行

systemctl mask firewalld
systemctl stop firewalld
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

#加载内核模块
modprobe dm_thin_pool

yum install -y docker
systemctl enable docker
systemctl start docker

# 准备 heketi 需要用到的磁盘
umount /dev/vdb3

注意：

• 如果分区被使用过，需要先清除分区表信息
  wipefs -a /dev/vdb3
• 如果擦除失败则可用dd命令暴力初始化，但在用dd命令之前请务必将两块磁盘unmount掉，否则你的磁盘会瞬间爆炸

安装 GLusterFS

docker run -d --name gluster -v /var/lib/glusterd:/var/lib/glusterd -v /dev:/dev --privileged --network host --restart always  gluster-centos:latest

docker exec -it gluster bash -c "echo root:password | chpasswd"

添加其他 gluster peer，仅在第一台 GlusterFS 节点上执行：
docker exec -it gluster gluster peer probe 172.19.35.71
docker exec -it gluster gluster peer probe 172.19.35.72

安装 Heketi

ssh-keygen
无脑回车即可

ssh-copy-id -i /root/.ssh/id_rsa -p 2222 root@192.168.21.39
ssh-copy-id -i /root/.ssh/id_rsa -p 2222 root@192.168.21.40
ssh-copy-id -i /root/.ssh/id_rsa -p 2222 root@192.168.21.41

准备配置文件：

mkdir -p /var/lib/heketi/config
vi /var/lib/heketi/config/heketi.json

heketi.json

{
  "_port_comment": "Heketi Server Port Number",
  "port": "8080",

  "_enable_tls_comment": "Enable TLS in Heketi Server",
  "enable_tls": false,

  "_cert_file_comment": "Path to a valid certificate file",
  "cert_file": "",

  "_key_file_comment": "Path to a valid private key file",
  "key_file": "",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": false,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "_key_comment": "Set the admin key in the next line",
      "key": ""
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "_key_comment": "Set the user key in the next line",
      "key": ""
    }
  },

  "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
  "backup_db_to_kube_secret": false,

  "_profiling": "Enable go/pprof profiling on the /debug/pprof endpoints.",
  "profiling": false,

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/root/.ssh/id_rsa",
      "user": "root",
      "port": "2222",
      "fstab": "/etc/fstab",
      "backup_lvm_metadata": false,
      "debug_umount_failures": true
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

     "_refresh_time_monitor_gluster_nodes": "Refresh time in seconds to monitor Gluster nodes",
    "refresh_time_monitor_gluster_nodes": 120,

    "_start_time_monitor_gluster_nodes": "Start time in seconds to monitor Gluster nodes when the heketi comes up",
    "start_time_monitor_gluster_nodes": 10,

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "debug",

    "_auto_create_block_hosting_volume": "Creates Block Hosting volumes automatically if not found or exsisting volume exhausted",
    "auto_create_block_hosting_volume": true,

    "_block_hosting_volume_size": "New block hosting volume will be created in size mentioned, This is considered only if auto-create is enabled.",
    "block_hosting_volume_size": 500,

    "_block_hosting_volume_options": "New block hosting volume will be created with the following set of options. Removing the group gluster-block option is NOT recommended. Additional options can be added next to it separated by a comma.",
    "block_hosting_volume_options": "group gluster-block",

    "_pre_request_volume_options": "Volume options that will be applied for all volumes created. Can be overridden by volume options in volume create request.",
    "pre_request_volume_options": "",

    "_post_request_volume_options": "Volume options that will be applied for all volumes created. To be used to override volume options in volume create request.",
    "post_request_volume_options": ""
  }
}

编辑 topology 文件
vi /var/lib/heketi/config/topology.json

{
  "clusters": [
    {
      "nodes": [
        {
          "node": {
            "hostnames": {
              "manage": [
                "172.19.35.70"
              ],
              "storage": [
                "172.19.35.70"
              ]
            },
            "zone": 1
          },
          "devices": [
            {
              "name": "/dev/vdb3",
              "destroydata": false
            }
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "172.19.35.71"
              ],
              "storage": [
                "172.19.35.71"
              ]
            },
            "zone": 1
          },
          "devices": [
            {
              "name": "/dev/vdb3",
              "destroydata": false
            }
          ]
        },
        {
          "node": {
            "hostnames": {
              "manage": [
                "172.19.35.72"
              ],
              "storage": [
                "172.19.35.72"
              ]
            },
            "zone": 1
          },
          "devices": [
            {
              "name": "/dev/vdb3",
              "destroydata": false
            }
          ]
        }
      ]
    }
  ]
}

启动 Heketi

docker run -d --network host --restart always --name heketi -v /var/lib/heketi:/var/lib/heketi -v /var/lib/heketi/config:/etc/heketi -v /root/.ssh:/root/.ssh cargo.dev.caicloud.xyz/release/heketi:9

docker exec -it heketi heketi-cli topology load --json=/etc/heketi/topology.json

查看集群列表

docker exec -it heketi heketi-cli cluster list