k8s glusterfs mysql_Kubernetes+GlusterFS使用用例

GlusterFS部署

▶ yum install centos-release-gluster -y

▶ yum install glusterfs-server

▶ mkfs.xfs /dev/vdd

▶ vim /etc/fstab

/dev/vdd /glusterfs xfs defaults 1 2

▶  mount -a && mount

▶ yum install glusterfs-server

▶ systemctl enable glusterd

▶ systemctl start glusterd

▶ systemctl status glusterd

▶ gluster peer probe sibat-kubernetes-02

▶ gluster peer probe sibat-kubernetes-03

▶ gluster peer probe sibat-kubernetes-04

▶ gluster peer probe sibat-kubernetes-05

▶  gluster peer status

▶ gluster volume create k8s-data replica 5 transport tcp sibat-kubernetes-01:/gluster sibat-kubernetes-02:/gluster sibat-kubernetes-03:/gluster sibat-kubernetes-04:/gluster sibat-kubernetes-05:/gluster force

▶ gluster volume list

k8s-data

▶ gluster volume info

Volume Name: k8s-data

Type: Replicate

Volume ID: 658b9a86-6cf8-4dcc-a8b8-e581381d5608

Status: Created

Snapshot Count: 0

Number of Bricks: 1 x 5 = 5

Transport-type: tcp

Bricks:

Brick1: sibat-kubernetes-01:/gluster

Brick2: sibat-kubernetes-02:/gluster

Brick3: sibat-kubernetes-03:/gluster

Brick4: sibat-kubernetes-04:/gluster

Brick5: sibat-kubernetes-05:/gluster

Options Reconfigured:

transport.address-family: inet

storage.fips-mode-rchecksum: on

nfs.disable: on

performance.client-io-threads: off

▶  gluster volume start k8s-data

volume start: k8s-data: success

▶  gluster volume quota k8s-data enable

volume quota : success

▶ gluster volume quota k8s-data limit-usage / 10000GB

volume quota : success

heketi

▶ yum install -y heketi heketi-client

▶ cat /etc/heketi/heketi.json

{

"port": "18080",

"use_auth": true,

"jwt": {

"admin": {

"key": "brunutRaspuWRe1404"

},

"user": {

"key": "brunutR2020"

}

},

"glusterfs": {

"executor": "ssh",

"sshexec": {

"keyfile": "/etc/heketi/heketi_key",

"user": "root",

"port": "22",

"fstab": "/etc/fstab"

},

"db": "/var/lib/heketi/heketi.db",

"loglevel" : "debug"

}

}

▶ cat /etc/heketi/topolgy_demo.json

{

"clusters": [

{

"nodes": [

{

"node": {

"hostnames": {

"manage": [

"sibat-kubernetes-1"

],

"storage": [

"192.168.233.11"

]

},

"zone": 1

},

"devices": [

"/dev/vdc"

]

},

{  "node": {

"hostnames": {

"manage": [

"sibat-kubernetes-2"

],

"storage": [

"192.168.233.212"

]

},

"zone": 1

},

"devices": [

"/dev/vdc"

]

},

{ "node": {

"hostnames": {

"manage": [

"sibat-kubernetes-3"

],

"storage": [

"192.168.233.108"

]

},

"zone": 1

},

"devices": [

"/dev/vdc"

]

},

{ "node": {

"hostnames": {

"manage": [

"sibat-kubernetes-4"

],

"storage": [

"192.168.233.64"

]

},

"zone": 1

},

"devices": [

"/dev/vdc"

]

},

{ "node": {

"hostnames": {

"manage": [

"sibat-kubernetes-5"

],

"storage": [

"192.168.233.96"

]

},

"zone": 1

},

"devices": [

"/dev/vdc"

]

}

]

}

]

}

▶  heketi-cli --server [http://192.168.233.247:18080](http://192.168.233.247:18080/) --user admin --secret brunutRaspuWRe1404 topology load --json=/etc/heketi/topolgy.json

Found node sibat-kubernetes-01 on cluster 0c085268e5bc20f7ac434d6aaddc4ca6

Adding device /dev/vdd ... OK

Found node sibat-kubernetes-03 on cluster 0c085268e5bc20f7ac434d6aaddc4ca6

Adding device /dev/vdd ... OK

Found node sibat-kubernetes-02 on cluster 0c085268e5bc20f7ac434d6aaddc4ca6

Adding device /dev/vdd ... Unable to add device: Setup of device /dev/vdd failed (already initialized or contains data?):   Device /dev/vdd not found.

Found node sibat-kubernetes-04 on cluster 0c085268e5bc20f7ac434d6aaddc4ca6

Adding device /dev/vdd ... OK

Found node sibat-kubernetes-05 on cluster 0c085268e5bc20f7ac434d6aaddc4ca6

Adding device /dev/vdd ... OK

▶ ssh-keygen -f /etc/heketi/heketi\_key -t rsa -N ''

Generating public/private rsa key pair.

Your identification has been saved in /etc/heketi/heketi\_key.

Your public key has been saved in /etc/heketi/heketi\_key.pub.

The key fingerprint is:

SHA256:KphU9aFUhWCzm75QA1grMIm6MWILFoMGVPDAQcBYYFM root@sibat-kubernetes-1

The key's randomart image is:

+---\[RSA 2048\]----+

|^%\*E. \*ooo.      |

|\*O+o = =..       |

|o =.+ o .        |

|\*o o . o         |

|=+o   = S        |

|.o o o o         |

|  o o o          |

|     o .         |

|      .          |

+----\[SHA256\]-----+

2020年 04月 14日 星期二 23:32:05 EDT sibat-kubernetes-1 root:~

▶ chown heketi:heketi  /etc/heketi/heketi\*

▶ heketi-cli cluster info 0c085268e5bc20f7ac434d6aaddc4ca6

Cluster id: 0c085268e5bc20f7ac434d6aaddc4ca6

Nodes:

5d27f7967a32032fc7343ef51f9c139e

64f24f93443adbccb47bf0dc52a8ca85

987050d88911b2d8dec7faf796f88b76

a6d29c2fd347dae4a646cde5937c5dd6

a8f08e5c6fb6c1020796efcbcad9c06a

Volumes:

Block: true

File: true

▶ heketi-cli volume create --size=5

Name: vol_b91e90468865c4bf1518b6943882be5e

Size: 5

Volume Id: b91e90468865c4bf1518b6943882be5e

Cluster Id: 0c085268e5bc20f7ac434d6aaddc4ca6

Mount: 192.168.233.247:vol_b91e90468865c4bf1518b6943882be5e

Mount Options: backup-volfile-servers=192.168.233.69,192.168.233.142,192.168.233.194,192.168.233.64

Block: false

Free Size: 0

Reserved Size: 0

Block Hosting Restriction: (none)

Block Volumes: []

Durability Type: replicate

Distribute Count: 1

Replica Count: 3

创建StorageClass

▶ vim gluster-sc.yaml

apiVersion: storage.k8s.io/v1

kind: StorageClass

metadata:

name: glusterfs

annotations:

storageclass.beta.kubernetes.io/is-default-class: true

storageclass.kubernetes.io/is-default-class: true

provisioner: kubernetes.io/glusterfs

reclaimPolicy: Delete

parameters:

resturl: "http://192.168.233.11:18080"

restauthenabled: "true"

restuser: "admin"

restuserkey: "brunutRaspuWRe1404"

gidMin: "40000"

gidMax: "50000"

volumetype: "replicate:3"

使用用例

apiVersion: apps/v1

kind: StatefulSet

metadata:

name: jenkins

namespace: devops

labels:

name: jenkins

spec:

selector:

matchLabels:

name: jenkins

serviceName: jenkins

replicas: 1

updateStrategy:

type: RollingUpdate

template:

metadata:

name: jenkins

labels:

name: jenkins

spec:

terminationGracePeriodSeconds: 10

serviceAccountName: jenkins

containers:

- name: jenkins

image: jenkins/jenkins:lts

imagePullPolicy: Always

ports:

- containerPort: 8080

- containerPort: 50000

resources:

limits:

cpu: 4

memory: 8Gi

requests:

cpu: 0.5

memory: 500Mi

env:

- name: LIMITS_MEMORY

valueFrom:

resourceFieldRef:

resource: limits.memory

divisor: 1Mi

- name: JAVA_OPTS

# value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85

value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85

- name: JENKINS_OPTS

value: --prefix=/jenkins

volumeMounts:

- name: jenkins-home

mountPath: /var/jenkins_home

livenessProbe:

httpGet:

path: /jenkins/login

port: 8080

initialDelaySeconds: 60

timeoutSeconds: 5

failureThreshold: 12 # ~2 minutes

readinessProbe:

httpGet:

path: /jenkins/login

port: 8080

initialDelaySeconds: 60

timeoutSeconds: 5

failureThreshold: 12 # ~2 minutes

securityContext:

fsGroup: 1000

volumeClaimTemplates:

- metadata:

name: jenkins-home

annotations:

volume.beta.kubernetes.io/storage-class: glusterfs

spec:

accessModes: [ "ReadWriteOnce" ]

resources:

requests:

storage: 100Gi

---

apiVersion: v1

kind: Service

metadata:

name: jenkins

namespace: devops

spec:

type: NodePort

selector:

name: jenkins

# ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)

#externalTrafficPolicy: Local

ports:

-

name: http

port: 80

targetPort: 8080

nodePort: 32000

protocol: TCP

-

name: agent

port: 50000

protocol: TCP

PVC使用实例

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

name: jnlp-mvn

namespace: devops

spec:

accessModes:

- ReadWriteMany

storageClassName: glusterfs

resources:

requests:

storage: 50Gi