k8s安装rabbitmq 集群

GitHub地址：https://github.com/Idiomroot/k8s-rabbitmq.git

一、下载插件

#mkdir plugins && cd plugins

#wget https://github.com/rabbitmq/rabbitmq-autocluster/releases/download/0.10.0/autocluster-0.10.0.ez

#wget https://github.com/rabbitmq/rabbitmq-autocluster/releases/download/0.10.0/rabbitmq_aws-0.10.0.ez

二、创建Dockerfile

#cd ../

#vi Dockerfile
FROM rabbitmq:3.6.8
MAINTAINER Idiom
ENV RABBITMQ_USE_LONGNAME=true \
    AUTOCLUSTER_LOG_LEVEL=debug \
    AUTOCLUSTER_CLEANUP=true \
    CLEANUP_INTERVAL=60 \
    CLEANUP_WARN_ONLY=false \
    AUTOCLUSTER_TYPE=k8s \
    LANG=en_US.UTF-8
ADD plugins/*.ez /usr/lib/rabbitmq/lib/rabbitmq_server-3.6.8/plugins/
RUN rabbitmq-plugins enable --offline autocluster
RUN rabbitmq-plugins enable --offline rabbitmq_management

#docker build -t 172.16.0.14:5000/pointsmart/rabbitmq3.7:v4 .

#docker push 172.16.0.14:5000/pointsmart/rabbitmq3.7:v4 （没有私有harbor可以省略此步骤）

 

三、生成cookie文件

# vi erlang-cookie.sh

#! /bin/bash

echo $(openssl rand -base64 32) > erlang.cookie

kubectl create secret generic erlang.cookie --from-file=erlang.cookie

#sh erlang-cookie.sh

四、配置rbac

# vi rabbitmq-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rabbitmq
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: endpoint-reader
rules:
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: endpoint-reader
subjects:
- kind: ServiceAccount
  name: rabbitmq
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: endpoint-reader

五、编写service

#vi serveice-rabbitmq.yaml
kind: Service
apiVersion: v1
metadata:
  name: rabbitmq-service
spec:
  type: NodePort
  ports:
    - name: mangement
      protocol: TCP
      port: 15672
      nodePort: 32001
    - name: smp 
      protocol: TCP
      port: 5672
      nodePort: 32002
  selector:
    app: rabbitmq
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq
  labels:
    app: rabbitmq
spec:
  clusterIP: None
  ports:
  - port: 5672
    name: amqp
  selector:
    app: rabbitmq

六、编写StatefulSet

# vi StatefulSet.yaml
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: rabbitmq
spec:
  serviceName: rabbitmq
  replicas: 3
  template:
    metadata:
      labels:
        app: rabbitmq
    spec:
      serviceAccountName: rabbitmq
      imagePullSecrets: 
        - name: regsecret
      containers:
      - name: rabbitmq
        image: 172.16.0.14:5000/pointsmart/rabbitmq3.7:v4 
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            memory: "256Mi"
            cpu: "150m"
          limits:
            memory: "512Mi"
            cpu: "250m"
        volumeMounts:
          - name: rabbitmq-data
            mountPath: /var/lib/rabbitmq/mnesia
        ports:
        - containerPort: 5672
          name: amqp
        env:
          - name: RABBITMQ_DEFAULT_USER
            value: comma 
          - name: RABBITMQ_DEFAULT_PASS
            value: commaai2017
          - name: RABBITMQ_ERLANG_COOKIE
            valueFrom:
              secretKeyRef:
                name: erlang.cookie
                key: erlang.cookie
          - name: MY_POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: K8S_SERVICE_NAME
            value: "rabbitmq"
          - name: RABBITMQ_USE_LONGNAME
            value: "true"
          - name: RABBITMQ_NODENAME
            value: "rabbit@$(MY_POD_NAME).$(K8S_SERVICE_NAME)"
          - name: RABBITMQ_NODE_TYPE
            value: disc
          - name: AUTOCLUSTER_TYPE
            value: "k8s"
          - name: AUTOCLUSTER_DELAY
            value: "10"
          - name: AUTOCLUSTER_CLEANUP
            value: "true"
          - name: CLEANUP_WARN_ONLY
            value: "false"
          - name: K8S_ADDRESS_TYPE
            value: "hostname"
          - name: K8S_HOSTNAME_SUFFIX
            value: ".$(K8S_SERVICE_NAME)"
      volumes:
        - name: rabbitmq-data
          persistentVolumeClaim:
            claimName: rabbitmq-data-claim
#vi pvc-data.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rabbitmq-data-claim
  annotations:
    volume.beta.kubernetes.io/storage-class: "pointsmart-nfs-storage"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 30Gi
如果没有存储类，则可以直接创建PV和PVC
vi  nfs-data.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
    name: rabbitmq-data
    labels:
      release: rabbitmq-data
spec:
    capacity:
      storage: 60Gi
    accessModes:
      - ReadWriteMany
    persistentVolumeReclaimPolicy: Delete
    nfs:
      path: /rabbit #这个文件夹要提前在nfs下面创建好
      server: nfs地址
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rabbitmq-data-claim
spec:
  accessModes:
    - ReadWriteMany
  resources:  
    requests:
      storage: 60Gi
  selector:
    matchLabels:
      release: rabbitmq-data

#kubectl create -f .

## kubectl get pod -o wide| grep rabbitmq

rabbitmq-0 1/1 Running 0 126m 172.30.28.2 172.16.0.9 <none> <none>

rabbitmq-1 1/1 Running 0 126m 172.30.3.2 172.16.0.8 <none> <none>

rabbitmq-2 1/1 Running 0 86m 172.30.28.7 172.16.0.9 <none> <none>

[root@bdy-master1 rabbitmq]# kubectl describe service rabbitmq

Name: rabbitmq

Namespace: default

Labels: app=rabbitmq

Annotations: <none>

Selector: app=rabbitmq

Type: ClusterIP

IP: None

Port: amqp 5672/TCP

TargetPort: 5672/TCP

Endpoints: 172.30.28.2:5672,172.30.28.7:5672,172.30.3.2:5672

Session Affinity: None

Events: <none>

[root@bdy-master1 rabbitmq]# kubectl describe service rabbitmq-service

Name: rabbitmq-service

Namespace: default

Labels: <none>

Annotations: <none>

Selector: app=rabbitmq

Type: NodePort

IP: 10.254.216.47

Port: mangement 15672/TCP

TargetPort: 15672/TCP

NodePort: mangement 32001/TCP

Endpoints: 172.30.28.2:15672,172.30.28.7:15672,172.30.3.2:15672

Port: smp 5672/TCP

TargetPort: 5672/TCP

NodePort: smp 32002/TCP

Endpoints: 172.30.28.2:5672,172.30.28.7:5672,172.30.3.2:5672

Session Affinity: None

External Traffic Policy: Cluster

Events: <none>

网页访问：http://nodeip:32001

 

注意：如果启动后出现报错： 

=INFO REPORT==== 8-Aug-2019::08:09:41 ===

autocluster: (cleanup) No partitioned nodes found.

解决办法：需要将其他两个节点加入到第一个节点中

#kubectl exec -it rabbitmq-xxxxx /bin/bash 

root@rabbitmq-xxxxx:/#rabbitmqctl stop_app

root@rabbitmq-xxxxx:/#rabbitmqctl join_cluster  rabbit@rabbitmq-0

root@rabbitmq-xxxxx:/#rabbitmqctl start_app

若出现节点无法加入集群的问题

root@rabbitmq-xxxxx:/#rabbitmqctl reset

root@rabbitmq-xxxxx:/#rabbitmqctl join_cluster  rabbit@rabbitmq-0

root@rabbitmq-xxxxx:/#rabbitmqctl start_app