部署glusterfs及Heketi

部署GlusterFS集群

• 由gfs01.lampnick.com,gfs02.lampnick.com,gfs03.lampnick.com三个节点组成GlusterFS存储集群，并将gfs01.lampnick.com节点部署为heketi服务器。在各节点上，均使用sda用于为GlusterFS提供存储空间
• 分别在三个节点上安装glusterfs-server程序包，并启动glusterfsd服务

	[root@worker1 ~]#  yum install centos-release-gluster -y
	[root@worker1 ~]# yum --enablerepo=centos-gluster*-test -y install glusterfs-server
	[root@worker1 ~]# systemctl start glusterd.service && systemctl enable glusterd.service

• 修改各节点hosts文件

	[root@worker1 ~]# vim /etc/hosts
	加入如下内容
	10.5.24.223 gfs01.lampnick.com
	10.5.24.224 gfs02.lampnick.com
	10.5.24.225 gfs03.lampnick.com

• 在节点01上使用"glusterfs peer probe"命令发现其他节点，组建GlusterFS集群

	[root@worker1 ~]# gluster peer probe gfs02.lampnick.com
	peer probe: success. 
	[root@worker1 ~]# gluster peer probe gfs03.lampnick.com
	peer probe: success. 

• 使用节点状态命令gluster peer status确认各节点已经加入同一个可信池中(trusted pool)

	[root@worker1 ~]# gluster peer status
	Number of Peers: 2

	Hostname: gfs02.lampnick.com
	Uuid: 59905d22-494c-4ff0-b151-3a9d0a48ba29
	State: Peer in Cluster (Connected)

	Hostname: gfs03.lampnick.com
	Uuid: e9999498-d37b-4f85-a7b0-316fbcae4ff6
	State: Peer in Cluster (Connected)

部署Heketi:将gfs01.lampnick.com节点部署为heketi服务器,以下命令均在gfs01.lampnick.com节点上运行

• 安装Heketi

	[root@worker1 ~]# yum -y install heketi heketi-client

• 配置Heketi用户能够基于SSH密钥的认证方式连接到GlusterFS集群中的各节点，并拥有相应节点的管理权限

	[root@worker1 ~]# ssh-keygen -f /etc/heketi/heketi_key -t rsa -N ''
	[root@worker1 ~]# chown heketi:heketi /etc/heketi/heketi_key*
	[root@worker1 ~]# for host in gfs01.lampnick.com gfs02.lampnick.com gfs03.lampnick.com; do ssh-copy-id -i /etc/heketi/heketi_key.pub root@${host};done

• 设置Heketi的主配置文件/etc/heketi/heketi.json，如下是默认配置

	{
	  "_port_comment": "Heketi Server Port Number",
	  "port": "8080",

	  "_use_auth": "Enable JWT authorization. Please enable for deployment",
	  "use_auth": false,

	  "_jwt": "Private keys for access",
	  "jwt": {
		"_admin": "Admin has access to all APIs",
		"admin": {
		  "key": "My Secret"
		},
		"_user": "User only has access to /volumes endpoint",
		"user": {
		  "key": "My Secret"
		}
	  },

	  "_glusterfs_comment": "GlusterFS Configuration",
	  "glusterfs": {
		"_executor_comment": [
		  "Execute plugin. Possible choices: mock, ssh",
		  "mock: This setting is used for testing and development.",
		  "      It will not send commands to any node.",
		  "ssh:  This setting will notify Heketi to ssh to the nodes.",
		  "      It will need the values in sshexec to be configured.",
		  "kubernetes: Communicate with GlusterFS containers over",
		  "            Kubernetes exec api."
		],
		"executor": "mock",

		"_sshexec_comment": "SSH username and private key file information",
		"sshexec": {
		  "keyfile": "path/to/private_key",
		  "user": "sshuser",
		  "port": "Optional: ssh port.  Default is 22",
		  "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
		},

		"_kubeexec_comment": "Kubernetes configuration",
		"kubeexec": {
		  "host" :"https://kubernetes.host:8443",
		  "cert" : "/path/to/crt.file",
		  "insecure": false,
		  "user": "kubernetes username",
		  "password": "password for kubernetes user",
		  "namespace": "OpenShift project or Kubernetes namespace",
		  "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
		},

		"_db_comment": "Database file name",
		"db": "/var/lib/heketi/heketi.db",

		"_loglevel_comment": [
		  "Set log level. Choices are:",
		  "  none, critical, error, warning, info, debug",
		  "Default is warning"
		],
		"loglevel" : "debug"
	  }
	}

• 启动Heketi及设置开机启动

	[root@worker1 ~]# systemctl enable heketi && systemctl start heketi

• 测试heketi

	[root@worker1 ~]# curl http://gfs01.lampnick.com:8080/hello
	Hello from Heketi

设置Heketi系统拓扑

• 编写配置文件放在/etc/heketi/topology.json中

	{
	  "clusters": [{
		"nodes": [{
			"node": {
			  "hostnames": {
				"manage": [
				  "10.5.24.223"
				],
				"storage": [
				  "10.5.24.223"
				]
			  },
			  "zone": 1
			},
			"devices": [
			  "/dev/sdb",
			  "/dev/sdc",
			  "/dev/sdd"
			]
		  },
		  {
			"node": {
			  "hostnames": {
				"manage": [
				  "10.5.24.224"
				],
				"storage": [
				  "10.5.24.224"
				]
			  },
			  "zone": 1
			},
			"devices": [
			  "/dev/sdb",
			  "/dev/sdc",
			  "/dev/sdd"
			]
		  },
		  {
			"node": {
			  "hostnames": {
				"manage": [
				  "10.5.24.225"
				],
				"storage": [
				  "10.5.24.225"
				]
			  },
			  "zone": 1
			},
			"devices": [
			  "/dev/sdb",
			  "/dev/sdc",
			  "/dev/sdd"
			]
		  }
		]
	  }]
	}

• 运行如下命令加载拓扑信息，完成集群配置。如下命令会生成一个集群，并为其添加的各节点生成随机ID

	[root@worker1 ~]# export HEKETI_CLI_SERVRR=http://gfs01.lampnick.com:8080
	[root@worker1 heketi]# heketi-cli topology load --json=topology.json
	Creating cluster ... ID: a2382eaaab98f3feb55961b37c7fb503
		Allowing file volumes on cluster.
		Allowing block volumes on cluster.
		Creating node 10.5.24.223 ... ID: 2b717655a23dca54bf113dc49b2eb0a3
			Adding device /dev/sda ... OK
		Creating node 10.5.24.224 ... ID: 8295c7fd9819f48284251361b23d8af5
			Adding device /dev/sda ... OK
		Creating node 10.5.24.225 ... ID: c22123f193de86d67c1fb6a808310bb6
			Adding device /dev/sda ... OK

• 运行如下命令查看集群状态

	[root@worker1 heketi]# heketi-cli cluster info a2382eaaab98f3feb55961b37c7fb503
	Cluster id: a2382eaaab98f3feb55961b37c7fb503
	Nodes:
	2b717655a23dca54bf113dc49b2eb0a3
	8295c7fd9819f48284251361b23d8af5
	c22123f193de86d67c1fb6a808310bb6
	Volumes:

	Block: true

	File: true

• 使用 heketi-cli volume create --size= [options]能创建存储卷

	[root@worker1 heketi]# heketi-cli volume create --size=20
	Name: vol_4ba7b17e5839c1e5d71a1067339941a2
	Size: 20
	Volume Id: 4ba7b17e5839c1e5d71a1067339941a2
	Cluster Id: a2382eaaab98f3feb55961b37c7fb503
	Mount: 10.5.24.223:vol_4ba7b17e5839c1e5d71a1067339941a2
	Mount Options: backup-volfile-servers=10.5.24.224,10.5.24.225
	Block: false
	Free Size: 0
	Reserved Size: 0
	Block Hosting Restriction: (none)
	Block Volumes: []
	Durability Type: replicate
	Distributed+Replica: 3

– 删除Heketi卷的命令为heketi-cli volume delete

	[root@worker1 heketi]# heketi-cli volume delete 4ba7b17e5839c1e5d71a1067339941a2
	Volume 4ba7b17e5839c1e5d71a1067339941a2 deleted

• 一个支持动态动态存储卷配置的GlusterFS存储集群设置完成。

使用手动方式在k8s中存储卷

• 手动创建GlusterFS卷

    # 在所有节点执行
    [root@worker1 ~]# mkdir -p /data/brick/kube-glusterfs
    
    # 在gfs01.lampnick.com执行
    [root@worker1 ~]# gluster volume create kube-glusterfs replica 3 gfs01.lampnick.com:/data/brick/kube-glusterfs gfs02.lampnick.com:/data/brick/kube-glusterfs gfs03.lampnick.com:/data/brick/kube-glusterfs force
    volume create: kube-glusterfs: success: please start the volume to access data
    
    [root@worker1 ~]# gluster volume start kube-glusterfs
    volume start: kube-glusterfs: success
    
    [root@worker1 ~]# gluster volume info
     
    Volume Name: kube-glusterfs
    Type: Replicate
    Volume ID: 4be17a6f-3dde-402f-80e6-c9b7e6ebd164
    Status: Started
    Snapshot Count: 0
    Number of Bricks: 1 x 3 = 3
    Transport-type: tcp
    Bricks:
    Brick1: gfs01.lampnick.com:/data/brick/kube-glusterfs
    Brick2: gfs02.lampnick.com:/data/brick/kube-glusterfs
    Brick3: gfs03.lampnick.com:/data/brick/kube-glusterfs
    Options Reconfigured:
    transport.address-family: inet
    nfs.disable: on
    performance.client-io-threads: off

• 创建endpoints

    apiVersion: v1
    kind: Endpoints
    metadata:
      name: glusterfs-endpoints
      namespace: dev
    subsets:
    - addresses:
      - ip: 10.5.24.223
      ports:
      - port: 24007
        name: glusterd
    - addresses:
      - ip: 10.5.24.224
      ports:
      - port: 24007
        name: glusterd
    - addresses:
      - ip: 10.5.24.225
      ports:
      - port: 24007
        name: glusterd

• 在pod中配置

    volumeMounts:
    - name: glusterfsdata
      mountPath: /glusterdata
  volumes:
  - name: glusterfsdata
    glusterfs:
      endpoints: glusterfs-endpoints
      path: kube-glusterfs
      readOnly: false

本文摘自：《kubernetes进阶实战》-马哥