k8s持久化安装rabbitmq集群（李作强）

1、编写yaml文件

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-cluster-ss.yaml

kind: StatefulSet
apiVersion: apps/v1beta1
metadata:
  labels:
    app: rmq-cluster
  name: rmq-cluster
  namespace: public-service
spec:
  replicas: 3
  selector:
    matchLabels:
      app: rmq-cluster
  serviceName: rmq-cluster
  template:
    metadata:
      labels:
        app: rmq-cluster
    spec:
      containers:
      - args:
        - -c
        - cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
          rabbitmq-server
        command:
        - sh
        env:
        - name: RABBITMQ_DEFAULT_USER
          valueFrom:
            secretKeyRef:
              key: username
              name: rmq-cluster-secret
        - name: RABBITMQ_DEFAULT_PASS
          valueFrom:
            secretKeyRef:
              key: password
              name: rmq-cluster-secret
        - name: RABBITMQ_ERLANG_COOKIE
          valueFrom:
            secretKeyRef:
              key: cookie
              name: rmq-cluster-secret
        - name: K8S_SERVICE_NAME
          value: rmq-cluster
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: RABBITMQ_USE_LONGNAME
          value: "true"
        - name: RABBITMQ_NODENAME
          value: rabbit@$(POD_NAME).rmq-cluster.$(POD_NAMESPACE).svc.cluster.local
        - name: RABBITMQ_CONFIG_FILE
          value: /var/lib/rabbitmq/rabbitmq.conf
        image: reg.raykite.com/ops/rabbitmq:3.7-management
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - rabbitmqctl
            - status
          initialDelaySeconds: 30
          timeoutSeconds: 10
        name: rabbitmq
        ports:
        - containerPort: 15672
          name: http
          protocol: TCP
        - containerPort: 5672
          name: amqp
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - rabbitmqctl
            - status
          initialDelaySeconds: 10
          timeoutSeconds: 10
        volumeMounts:
        - mountPath: /etc/rabbitmq
          name: config-volume
          readOnly: false
        - mountPath: /var/lib/rabbitmq
          name: rabbitmq-storage
          readOnly: false
      serviceAccountName: rmq-cluster
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          items:
          - key: rabbitmq.conf
            path: rabbitmq.conf
          - key: enabled_plugins
            path: enabled_plugins
          name: rmq-cluster-config
        name: config-volume
  volumeClaimTemplates:
  - metadata:
      name: rabbitmq-storage
    spec:
      accessModes:
      - ReadWriteMany
      storageClassName: "rmq-storage-class"
      resources:
        requests:
          storage: 4Gi
 

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-configmap.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  name: rmq-cluster-config
  namespace: public-service
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
    enabled_plugins: |
      [rabbitmq_management,rabbitmq_peer_discovery_k8s].
    rabbitmq.conf: |
      loopback_users.guest = false
      ##Clustering
      cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
      cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
      cluster_formation.k8s.address_type = hostname
      #################################################
      # public-service is rabbitmq-cluster's namespace#
      #################################################
      cluster_formation.k8s.hostname_suffix = .rmq-cluster.public-service.svc.cluster.local
      cluster_formation.node_cleanup.interval = 10
      cluster_formation.node_cleanup.only_log_warning = true
      cluster_partition_handling = autoheal
      ## queue master locator
      queue_master_locator=min-masters
 

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-rmq-1
spec:
  capacity:
    storage: 4Gi
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "rmq-storage-class"
  nfs:
    # real share directory
    path: /data/k8s/rabbitmq-cluster-1
    # nfs real ip
    server: 10.10.10.237
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-rmq-2
spec:
  capacity:
    storage: 4Gi
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "rmq-storage-class"
  nfs:
    # real share directory
    path: /data/k8s/rabbitmq-cluster-2
    # nfs real ip
    server: 10.10.10.237
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-rmq-3
spec:
  capacity:
    storage: 4Gi
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "rmq-storage-class"
  nfs:
    # real share directory
    path: /data/k8s/rabbitmq-cluster-3
    # nfs real ip
    server: 10.10.10.237
 

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: rmq-cluster
  namespace: public-service
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rmq-cluster
  namespace: public-service
rules:
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rmq-cluster
  namespace: public-service
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rmq-cluster
subjects:
- kind: ServiceAccount
  name: rmq-cluster
  namespace: public-service
 

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-secret.yaml

kind: Secret
apiVersion: v1
metadata:
  name: rmq-cluster-secret
  namespace: public-service
stringData:
  cookie: ERLANG_COOKIE
  password: RABBITMQ_PASS
  url: amqp://RABBITMQ_USER:RABBITMQ_PASS@rmq-cluster-balancer
  username: RABBITMQ_USER
type: Opaque
 

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-service-cluster.yaml

kind: Service
apiVersion: v1
metadata:
  labels:
    app: rmq-cluster
  name: rmq-cluster
  namespace: public-service
spec:
  clusterIP: None
  ports:
  - name: amqp
    port: 5672
    targetPort: 5672
  selector:
    app: rmq-cluster

[root@k8s-master01 rabbitmq-cluster]# vim rabbitmq-service-lb.yaml

kind: Service
apiVersion: v1
metadata:
  labels:
    app: rmq-cluster
    type: LoadBalancer
  name: rmq-cluster-balancer
  namespace: public-service
spec:
  ports:
  - name: http
    port: 15672
    protocol: TCP
    targetPort: 15672
    nodePort: 30045
  - name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
    nodePort: 30044
  selector:
    app: rmq-cluster
  type: NodePort
 

2、创建namespace

[root@k8s-master01 rabbitmq-cluster]# kubectl create namespace public-service

如果不使用public-service，需要更改所有yaml文件的public-service为你namespace。

[root@k8s-master01 rabbitmq-cluster]# sed -i "s#public-service#YOUR_NAMESPACE#g" *.yaml

 

3、创建持久化pv

此处采用的是静态PV方式，后端使用的是NFS，为了方便扩展可以使用动态PV较好。

[root@nfs rabbitmq-cluster-1]# vim /etc/exports

/k8s/rmq-cluster/rabbitmq-cluster-1/ *(rw,sync,no_subtree_check,no_root_squash)

/k8s/rmq-cluster/rabbitmq-cluster-2/ *(rw,sync,no_subtree_check,no_root_squash)

/k8s/rmq-cluster/rabbitmq-cluster-3/ *(rw,sync,no_subtree_check,no_root_squash)

 

[root@k8s-master01 rabbitmq-cluster]# pwd

/root/efk/rabbitmq-cluster

 

[root@k8s-master01 rabbitmq-cluster]# ls

rabbitmq-cluster-ss.yaml  rabbitmq-pv.yaml  rabbitmq-secret.yaml  rabbitmq-service-lb.yaml  rabbitmq-configmap.yaml  rabbitmq-rbac.yaml  rabbitmq-service-cluster.yaml

 

创建集群

[root@k8s-master01 rabbitmq-cluster]# kubectl apply -f .

 

4、查看

pods、pv、pvc

[root@k8s-master01 rabbitmq-cluster]# kubectl get pods -n public-service

 

[root@k8s-master01 rabbitmq-cluster]# kubectl get pv -n public-service 

[root@k8s-master01 rabbitmq-cluster]# kubectl get pvc -n public-service

 

services，此时使用的NodePort方式，可改为traefik模式

[root@k8s-master01 rabbitmq-cluster]# kubectl get services -n public-service

 

5、访问测试

　　通过NodePort的端口

　　账号密码：guest