k8s 配置 glusterFS 动态供给

部署环境

Host	IP	k8s 版本	glusterFS版本	heketi版本	heketi-client 版本
k8s-master1	192.168.10.1	1.20.0	9.5-1.el7	heketi-8.0.0-1	heketi-client-8.0.0-1
k8s-master2	192.168.10.2	1.20.0	9.5-1.el7	heketi-8.0.0-1	heketi-client-8.0.0-1
k8s-master3	192.168.10.3	1.20.0	9.5-1.el7	heketi-8.0.0-1	heketi-client-8.0.0-1

前置条件：
每个用来做GFS数据存储必须是一个设备块，不能是一个目录。如/dev/sdb，/dev/sdb1。这块需要提前规划

部署 glusterFS 集群

hosts信息

# /etc/hosts
192.168.10.1 k8s-master1
192.168.10.2 k8s-master2
192.168.10.3 k8s-master3

$ ntpdate time.windows.com   # 三个节点必须做时钟同步

# 配置yum源（k8s-master1-k8s-master3）
$ yum -y install centos-release-gluster9.noarch   # 下载GFS的第9个大版本的源，v6与当前k8s不兼容


# 安装GlusterFS（k8s-master1-k8s-master3）
$ yum -y install glusterfs glusterfs-fuse glusterfs-server

配置服务和集群

# 启动glusterfs服务（k8s-master1-k8s-master3）
$ systemctl enable glusterd
$ systemctl start glusterd

# 配置集群（将节点k8s-master1-k8s-master3加入集群）
$ gluster peer probe k8s-master1
    peer probe: success. Probe on localhost not needed
$ gluster peer probe k8s-master2
    peer probe: success.
$ gluster peer probe k8s-master3
    peer probe: success.
$ gluster peer status
Number of Peers: 2

Hostname: k8s-master2
Uuid: c08a504c-4a92-403b-ae86-48e51b1efd05
State: Peer in Cluster (Connected)

Hostname: k8s-master3
Uuid: 9bbc907a-3f60-49fd-9911-9cfed9bb0844
State: Peer in Cluster (Connected)

部署GlusterFS管理客户端heketi

节点初始化

# 每个节点都需要创建一个heketi用户，root用户起不来
adduser heketi
passwd heketi

# 添加到sudoers
# 增加可写权限/etc/sudoers
chmod u+w /etc/sudoers
# 添加sudo
# vim /etc/sudoers
# root    ALL=(ALL)       ALL下面增加
heketi    ALL=(ALL)       NOPASSWD: ALL
# 恢复/etc/sudoers
chmod u-w /etc/sudoers

# 服务节点免密授权
# login heketi
# 下面操作只需要在k8s-master1节点执行
su - heketi
ssh-keygen -t rsa -b 4096
ssh-copy-id -i heketi@k8s-master1
ssh-copy-id -i heketi@k8s-master2
ssh-copy-id -i heketi@k8s-master3

安装heketi

# 所有节点安装heketi-client
yum install -y heketi-client

# 服务节点安装heketi
# login heketi
yum install -y heketi
# 配置服务
cd /etc/heketi/
cp heketi.json  heketi.json_bak

heketi.json

{
  "_port_comment": "Heketi Server Port Number",
  "port": "18080",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": true,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "admin"   ## 与后面一致
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "admin"
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/home/heketi/.ssh/id_rsa",
      "user": "heketi",
      "port": "22",
      "sudo": true,
      "fstab": "/etc/fstab"
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "debug"
  }
}

# 开机启动
systemctl enable heketi

# 启动服务
systemctl start heketi

# 启动状态
systemctl status heketi

初始化GFS 复制卷集群

复制卷：所有组成卷的服务器中存放的内容都完全相同，类似RAID1。

# 创建集群
$ heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 --json  cluster create
{"id":"60d0c41c0b232906f90b528fbb58400a","nodes":[],"volumes":[],"block":true,"file":true,"blockvolumes":[]}

# 添加节点
$ heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 --json  node add --cluster "60d0c41c0b232906f90b528fbb58400a" --management-host-name 192.168.10.1 --storage-host-name 192.168.10.1 --zone 1
{"zone":1,"hostnames":{"manage":["192.168.10.1"],"storage":["192.168.10.1"]},"cluster":"60d0c41c0b232906f90b528fbb58400a","id":"918f73f5e17662abbcd0552445a53766","state":"online","devices":[]}

$ heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 --json  node add --cluster "60d0c41c0b232906f90b528fbb58400a" --management-host-name 192.168.10.2 --storage-host-name 192.168.10.2 --zone 1
{"zone":1,"hostnames":{"manage":["192.168.10.2"],"storage":["192.168.10.2"]},"cluster":"60d0c41c0b232906f90b528fbb58400a","id":"918f73f5e17662abbcd0552445a53766","state":"online","devices":[]}

$ heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 --json  node add --cluster "60d0c41c0b232906f90b528fbb58400a" --management-host-name 192.168.10.3 --storage-host-name 192.168.10.3 --zone 1
{"zone":1,"hostnames":{"manage":["192.168.10.3"],"storage":["192.168.10.3"]},"cluster":"60d0c41c0b232906f90b528fbb58400a","id":"918f73f5e17662abbcd0552445a53766","state":"online","devices":[]}

# 添加存储设备
$ heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 --json  node list
Id:46e7b325ce20528bf8160d8c65a3c652 Cluster:60d0c41c0b232906f90b528fbb58400a
Id:524481c119e6295117979bb80046f9c8 Cluster:60d0c41c0b232906f90b528fbb58400a
Id:918f73f5e17662abbcd0552445a53766 Cluster:60d0c41c0b232906f90b528fbb58400a

## sdb为设备块，不是目录
$ heketi-cli --user admin --secret admin --server http://k8s-master1:18080   device add --name "/dev/sdb"  --node 46e7b325ce20528bf8160d8c65a3c652
$ heketi-cli --user admin --secret admin --server http://k8s-master1:18080   device add --name "/dev/sdb"  --node 524481c119e6295117979bb80046f9c8
$ heketi-cli --user admin --secret admin --server http://k8s-master1:18080   device add --name "/dev/sdb"  --node 918f73f5e17662abbcd0552445a53766

# 查看集群拓扑
$ heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 topology info

Cluster Id: 60d0c41c0b232906f90b528fbb58400a

    File:  true
    Block: true

    Volumes:

	Name: vol_af2533d49e30b106c7c1181807c93ad1
	Size: 10
	Id: af2533d49e30b106c7c1181807c93ad1
	Cluster Id: 60d0c41c0b232906f90b528fbb58400a
	Mount: 192.168.10.2:vol_af2533d49e30b106c7c1181807c93ad1
	Mount Options: backup-volfile-servers=192.168.10.3,192.168.10.1
	Durability Type: replicate
	Replica: 3
	Snapshot: Enabled
	Snapshot Factor: 1.00

		Bricks:
			Id: 3dbf6dd1f57f119054c583489a29242d
			Path: /var/lib/heketi/mounts/vg_a161942c4ea242e92cb753a18abc0483/brick_3dbf6dd1f57f119054c583489a29242d/brick
			Size (GiB): 10
			Node: 524481c119e6295117979bb80046f9c8
			Device: a161942c4ea242e92cb753a18abc0483

			Id: 90dc64eea016285370a4df707f235371
			Path: /var/lib/heketi/mounts/vg_04041ffb900f9e8a2d73cdd9a32ed62a/brick_90dc64eea016285370a4df707f235371/brick
			Size (GiB): 10
			Node: 46e7b325ce20528bf8160d8c65a3c652
			Device: 04041ffb900f9e8a2d73cdd9a32ed62a

			Id: bf602a9741293e6f1cd5102a385f87d5
			Path: /var/lib/heketi/mounts/vg_c0307d49fd4e8b3a5bd2b4ed48c4dcd9/brick_bf602a9741293e6f1cd5102a385f87d5/brick
			Size (GiB): 10
			Node: 918f73f5e17662abbcd0552445a53766
			Device: c0307d49fd4e8b3a5bd2b4ed48c4dcd9


    Nodes:

	Node Id: 46e7b325ce20528bf8160d8c65a3c652
	State: online
	Cluster Id: 60d0c41c0b232906f90b528fbb58400a
	Zone: 1
	Management Hostnames: 192.168.10.2
	Storage Hostnames: 192.168.10.2
	Devices:
		Id:04041ffb900f9e8a2d73cdd9a32ed62a   Name:/dev/sdb            State:online    Size (GiB):99      Used (GiB):10      Free (GiB):89
			Bricks:
				Id:90dc64eea016285370a4df707f235371   Size (GiB):10      Path: /var/lib/heketi/mounts/vg_04041ffb900f9e8a2d73cdd9a32ed62a/brick_90dc64eea016285370a4df707f235371/brick

	Node Id: 524481c119e6295117979bb80046f9c8
	State: online
	Cluster Id: 60d0c41c0b232906f90b528fbb58400a
	Zone: 1
	Management Hostnames: 192.168.10.3
	Storage Hostnames: 192.168.10.3
	Devices:
		Id:a161942c4ea242e92cb753a18abc0483   Name:/dev/sdb            State:online    Size (GiB):99      Used (GiB):10      Free (GiB):89
			Bricks:
				Id:3dbf6dd1f57f119054c583489a29242d   Size (GiB):10      Path: /var/lib/heketi/mounts/vg_a161942c4ea242e92cb753a18abc0483/brick_3dbf6dd1f57f119054c583489a29242d/brick

	Node Id: 918f73f5e17662abbcd0552445a53766
	State: online
	Cluster Id: 60d0c41c0b232906f90b528fbb58400a
	Zone: 1
	Management Hostnames: 192.168.10.1
	Storage Hostnames: 192.168.10.1
	Devices:
		Id:c0307d49fd4e8b3a5bd2b4ed48c4dcd9   Name:/dev/sdb            State:online    Size (GiB):99      Used (GiB):10      Free (GiB):89
			Bricks:
				Id:bf602a9741293e6f1cd5102a385f87d5   Size (GiB):10      Path: /var/lib/heketi/mounts/vg_c0307d49fd4e8b3a5bd2b4ed48c4dcd9/brick_bf602a9741293e6f1cd5102a385f87d5/brick

GlusterFS 在 kubernetes 集群的应用

StorageClass

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gluster-heketi-storageclass   # 存储类名称
provisioner: kubernetes.io/glusterfs
reclaimPolicy: Retain
parameters:
  resturl: "http://192.168.10.1:18080"    # heketi server 地址
  restauthenabled: "true"  
  restuser: "admin"  # 要与上面heketi.json中配置一致
  restuserkey: "admin" # 要与上面heketi.json中配置一致
  gidMin: "2000"
  #secretName: "heketi-secret"
  #secretNamespace: "default"
  #volumetype: "none"
  volumetype: "replicate:3"   # 当前是3个副本数
  clusterid: "60d0c41c0b232906f90b528fbb58400a"  # 执行 heketi-cli  --user admin --secret admin --server http://k8s-master1:18080 --json  node list    可以看到
# 支持扩容
allowVolumeExpansion: true

test

kind: Deployment
apiVersion: apps/v1
metadata:
  name: demo-mode3-nginx
  labels:
    name: demo-mode3-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      name: demo-mode3-nginx
  template:
    metadata:
      labels:
        name: demo-mode3-nginx
    spec:
      containers:
        - name: nginx
          image: nginx
          ports:
            - containerPort: 80
          volumeMounts:
            - name: demo-mode3-nginx-vol
              mountPath: "/usr/share/nginx/html"
      volumes:
      - name: demo-mode3-nginx-vol
        persistentVolumeClaim:
          claimName: glusterfs-vol-pvc02
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: glusterfs-vol-pvc02  
  namespace: default
spec:
  storageClassName: gluster-heketi-storageclass   # 指定存储类名称
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi

$ kubectl get pvc,pv |grep glusterfs-vol-pvc02
persistentvolumeclaim/glusterfs-vol-pvc02                                                                                              Bound    pvc-7623c393-0f56-40cc-ae86-ab71c9900895   10Gi       RWX            gluster-heketi-storageclass   11m
persistentvolume/pvc-7623c393-0f56-40cc-ae86-ab71c9900895   10Gi       RWX            Retain           Bound      default/glusterfs-vol-pvc02                                                                                              gluster-heketi-storageclass            9m50s

$ kubectl get po |grep demo-mode3-nginx
demo-mode3-nginx-68df46746d-qhh4g                         1/1     Running   0          12m

pod,pvc 的状态不为 Pending 即正常。