centos安装glusterFS集群

使用yum安装GlusterFs Server

# 安装server
yum install -y centos-release-gluster
yum install -y glusterfs-server

# 开机自启
systemctl enable glusterd

# 启动服务
systemctl start glusterd

# 查看运行状态
systemctl status glusterd

使用yum安装Heketi

安装服务和客户端

yum install -y centos-release-gluster

# 安装Heketi Server和客户端
yum install -y heketi heketi-client

修改配置文件heketi.json

vi /etc/heketi/heketi.json

主要修改以下几个点：

port，use_auth，admin.key，user.key，executor和sshexec

{
  "_port_comment": "Heketi Server Port Number",
  "port": "8080",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": true,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "admin@123"
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "user@123"
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    "executor": "ssh",

    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key",
      "user": "root",
      "port": "22",
      "fstab": "/etc/fstab"
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "warning"
  }
}

设置heketi免密访问GlusterFS

# 生成ssh-key
ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N ""

# 将ssh-key copy到其他机器上
ssh-copy-id -i /etc/heketi/heketi_key.pub root@192.168.112.129
ssh-copy-id -i /etc/heketi/heketi_key.pub root@192.168.112.130
ssh-copy-id -i /etc/heketi/heketi_key.pub root@192.168.112.131

启动Heketi

systemctl enable heketi
systemctl start heketi
systemctl status heketi

验证是否正常 

curl http://localhost:8080/hello

初始化GlusterFS集群

创建topology.json

vi /etc/heketi/topology.json

{
    "clusters": [{
        "nodes": [{
            "node": {
                "hostnames": {
                    "manage": ["192.168.112.129"],
                    "storage": ["192.168.112.129"]
                },
                "zone": 1
            },
            "devices": ["/dev/sda3"]
        }, {
            "node": {
                "hostnames": {
                    "manage": ["192.168.112.130"],
                    "storage": ["192.168.112.130"]
                },
                "zone": 2
            },
            "devices": ["/dev/sda3"]
        }, {
            "node": {
                "hostnames": {
                    "manage": ["192.168.112.131"],
                    "storage": ["192.168.112.131"]
                },
                "zone": 3
            },
            "devices": ["/dev/sda3"]
        }]
    }]
}

 通过topology.json组建GlusterFS集群

前提需要上面设置的devices的盘必须是空的，否则初始化报错

如果提示签名已存在，可以使用wipefs -a /dev/sda3去删除签名

heketi-cli --server http://localhost:8080 --user admin --secret admin@123 topology load --json=/etc/heketi/topology.json

查看集群状态

heketi-cli --user admin --secret admin@123 topology info

在k8s集群中使用GlusterFS作为默认存储

创建gluster-sc

vi glusterfs-sc.yaml

 其中heketi-secret中的数据值为上面heketi的密码转换base64得来的，下面的clusterid在初始化集群之后会返回回来。resturl就是heketi的地址，volumetype: "replicate:2"是用来设置文件副本

apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret
  namespace: kube-system
type: kubernetes.io/glusterfs
data:
  key: "LWQgYWRtaW5AMTIzCg=="
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    storageclass.beta.kubernetes.io/is-default-class: "true"
    storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce","ReadOnlyMany","ReadWriteMany"]'
  name: glusterfs
parameters:
  clusterid: "69ca99bb60f76f520130d88340af6934"
  gidMax: "50000"
  gidMin: "40000"
  restauthenabled: "true"
  resturl: "http://192.168.112.129:8080"
  restuser: admin
  secretName: heketi-secret
  secretNamespace: kube-system
  volumetype: "replicate:2"
provisioner: kubernetes.io/glusterfs
reclaimPolicy: Delete
volumeBindingMode: Immediate
allowVolumeExpansion: true