k8s部署mongodb分片集群

配置节点label

apiVersion: v1
kind: Node
metadata:    
  name: host1
  labels:
    mongodb: "mongo-0"
---
apiVersion: v1
kind: Node
metadata:
  name: host2
  labels:
    mongodb: "mongo-1"
---
apiVersion: v1
kind: Node
metadata:
  name: host3
  labels:
    mongodb: "mongo-2"

配置keyfile和docker私有仓库

apiVersion: v1
data:
  .dockerconfigjson: eyJhdXRocyI6eyIxOTIuMTY4LjEzLjUiOnsidXNlcm5hbWUiOiJ6eWNsb3VkIiwicGFzc3dvcmQiOiJaaGl5dWFuQ2xvdWQxMjM0ISIsImF1dGgiOiJlbmxqYkc5MVpEcGFhR2w1ZFdGdVEyeHZkV1F4TWpNMElRPT0ifX19
kind: Secret
metadata:
  name: zlgdocker
  namespace: zlg-zws
type: kubernetes.io/dockerconfigjson
---
apiVersion: v1
data:
  keyfile.key: ZGNDTWd4T3B3dGdsNlpFeExialZxajhvMmFENjVwVGtPVFVxTG0vMGJXd1FvMUhUZmtUM0pFSXZSN1pYRk41ZQoyWDI0TTJ1L2xremN0UjdQRTB0WXRYelY4enVWRjdBM2hRUTlsdnBFakQwdzBDOE1zQUYrSndPSzVPeDZPVzlPCkp2TUM5WVZtS3RXV1drcUNCN3c1RkV0WUxkK1hmNkJGWW16TUFIdGtoWUhCQmp1aHcrZk9RSDU5UTdJZXNEZFcKZHhFOWFQZDVEZUxaSmhVY0lZVlkvQnFrbit3RG1BdUoxWmd0WjlGejg2eFVwYTdSSHROWE5Qa2gzekRDZzFqNApGRDlrVkFkODEybVhyZ0FqNlROVTJoSTVDY0ZJem0vNDUraW10QUxCTFE1VDVOb0R1UUNtYVBJaEMrc3JvaHNlCkdEbVY0T1llMVl6eXVIRWJ4dHpWeFNSU0xJSUZSdUdkdnFOYXp1bXQ4UTVBWStOYnprSTBhcENOMmZxR3RudzUKUWVMNnVlekkvUFJwaFdmS0dkVG9XWG9SRWpqdzdyRkxISEtBeExWVi9pblQzaCtkUm9LRmdJVk1JcGFuOFVoSAoxcTNSQVBmKzAvVjRtUW1UT2swZGV5eUVMWnp6SjJDRE1uWWFVQTNDTG5aWFhiUDJjVTJ6ZFBVcjJDT0dKTWVtCjFvaXhEWUdsampXTGlqYzZNcFJZY3BIWXVmcFUvaUl3WXVUZnBrZWZCR004Yzg0TitScCtCWjQ5Uk1FbmFHMnMKaGN6aVA5LzJNc0NpV1VZblljVWEyaENnOG9jbHVlQmJFeFMxeFF3d2xPNzA4dTFsVFVyNmdJek93d1hjTjlQcwo1dkFVLzlVQXJDWDNaNmRtdFdqblBBSmZqUU1XQUs1VldTamthczJJZWgrVURQUEVmNlYrempodzFVeVRoMExICmdWN0lPWHRNTERiaURvcXdDelBIb085cEEvYjVDWHYrdjlFMXZNdC9ueTBQREpKb0NnOFBMeHA4ZjA1WXJtWk4KZ2V2dHNibGJpaGd2Q2E3SUJiVmZRQSswUFl6cXRzZ1dHeE43K2hSNnNWS01KM2ZFVFA4R2pLa29aUDl1ZzJCcwpEQUQ1aGw4clFwdi9DQnA4a0Uvd3RXU1RrRWc5ZTY3dUxzMVQvYUxQdW12cVdYc2ZON2xya05zR2dBc3pCU0dICnFjcnA1VHBFTDQ1K08vQ1Z4M0RRNVhGZWpmWHdwczFVZ1ZsSDZsVzRydEQyZUcyVm5VbkdzSURJc2Ixa2UzbkcKNUVrM25BQWdFOUlHUGFYWlF0R2UwRVg5QWU0UUZJdDV6R3JycG5DSnNYZERXR250Cg==
kind: Secret
metadata:
  name: zlgmongo-keyfile
  namespace: zlg-zws
type: Opaque

config server

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zlgmongo-config
  namespace: zlg-zws
spec:
  serviceName: zlgmongo-config
  replicas: 1
  selector:
    matchLabels:
      app: zlgmongo-config
  template:
    metadata:
      labels:
        app: zlgmongo-config
    spec:
      imagePullSecrets:
        - name:  zlgdocker
      nodeSelector:
        mongodb: "mongo-0"
      containers:
      - name: zlgmongo-config
        image: 192.168.13.5/solar/zlgmongo:master
        imagePullPolicy: Always
        resources:
          limits:
            memory: "128Mi"
            cpu: "500m"
          requests:
            memory: "1Mi"
            cpu: "1m"
        env:
          - name: POD_NAME
            valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
        ports:
        - containerPort: 27019
        volumeMounts:
          - name: configdb
            mountPath:  /data/configdb
            subPathExpr: $(POD_NAME)
          - mountPath: /data/auth/keyfile.key
            name: keyfile
            subPath: keyfile.key
            readOnly: true
        command:
          - "mongod"
          - "--keyFile"
          - "/data/auth/keyfile.key"
          - "--configsvr"
          - "--replSet"
          - "config"
          - "--bind_ip_all"
      volumes:
        - name: configdb
          hostPath:
              path: /dockershare/zlgmongo/data/configdb
              type: DirectoryOrCreate
        - name: keyfile
          secret:
              secretName: zlgmongo-keyfile
              defaultMode: 0600 
              items:
                - key: keyfile.key
                  path: keyfile.key
---
apiVersion: v1
kind: Service
metadata:
  name: zlgmongo-config
  namespace: zlg-zws
spec:
  selector:
    app: zlgmongo-config
  clusterIP: None
  ports:
  - port: 27019
    targetPort: 27019
    name: "27019"

shard

配置3个shard分别是zlgmongo-shard-0、zlgmongo-shard-1、zlgmongo-shard-2对应nodeSelector为mongo-0、mongo-1、mongo-2

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zlgmongo-shard-0
  namespace: zlg-zws
spec:
  serviceName: zlgmongo-shard-0
  selector:
    matchLabels:
      app: zlgmongo-shard-0
  replicas: 1
  template:
    metadata:
      labels:
        app: zlgmongo-shard-0
    spec:
      imagePullSecrets:
      - name:  zlgdocker
      nodeSelector:
        mongodb: "mongo-0"
      containers:
      - name: zlgmongo-shard
        image: 192.168.13.5/solar/zlgmongo:master
        imagePullPolicy: Always
        resources:
          limits:
            memory: "4Gi"
            cpu: "4000m"
          requests:
            memory: "1Mi"
            cpu: "10m"
        env:
          - name: POD_NAME
            valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
        ports:
        - containerPort: 27018
        volumeMounts:
          - name: db
            mountPath:  /data/db
            subPathExpr: $(POD_NAME)
          - mountPath: /data/auth/keyfile.key
            name: keyfile
            subPath: keyfile.key
            readOnly: true
        command:
          - "mongod"
          - "--wiredTigerCacheSizeGB"
          - "3"
          - "--keyFile"
          - "/data/auth/keyfile.key"
          - "--shardsvr"
          - "--replSet"
          - "$(POD_NAME)"
          - "--bind_ip_all"
      volumes:
        - name: db
          hostPath:
              path: /dockershare/zlgmongo/data/db
              type: DirectoryOrCreate
        - name: keyfile
          secret:
              secretName: zlgmongo-keyfile
              defaultMode: 0600 
              items:
                - key: keyfile.key
                  path: keyfile.key
---
apiVersion: v1
kind: Service
metadata:
  name: zlgmongo-shard-0
  namespace: zlg-zws
spec:
  selector:
    app: zlgmongo-shard-0
  clusterIP: None
  ports:
  - port: 27018
    targetPort: 27018
    name: "27018"

mongos

apiVersion: apps/v1
kind: Deployment
metadata:
  name: zlgmongo
  namespace: zlg-zws
spec:
  selector:
    matchLabels:
      app: zlgmongo
  template:
    metadata:
      labels:
        app: zlgmongo
    spec:
      imagePullSecrets:
        - name:  zlgdocker
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - topologyKey: "kubernetes.io/hostname"
              labelSelector:
                  matchLabels:
                    app: zlgmongo
      containers:
      - name: zlgmongo
        image: 192.168.13.5/solar/zlgmongo:master
        imagePullPolicy: Always
        resources:
          limits:
            memory: "2Gi"
            cpu: "4000m"
          requests:
            memory: "64Mi"
            cpu: "10m"
        env:
          - name: DATADIR
            valueFrom:
                configMapKeyRef:
                  name: zlgconfigmap
                  key: datadir
          - name: SHAREDIR
            valueFrom:
                configMapKeyRef:
                  name: zlgconfigmap
                  key: sharedir
          - name: CONFD_PORT
            valueFrom:
                configMapKeyRef:
                  name: zlgconfigmap
                  key: confd-port
          - name: CONFD_NAMESPACE
            valueFrom:
                configMapKeyRef:
                  name: zlgconfigmap
                  key: confd-namespace
          - name: CONFD_TOKEN
            valueFrom:
                configMapKeyRef:
                  name: zlgconfigmap
                  key: confd-token
          - name: DEBUG
            valueFrom:
                configMapKeyRef:
                  name: zlgconfigmap
                  key: debug
        ports:
        - containerPort: 27017
        volumeMounts:
          - mountPath: /data/auth/keyfile.key
            name: keyfile
            subPath: keyfile.key
            readOnly: true
        command:
          - "mongos"
          - "--keyFile"
          - "/data/auth/keyfile.key"
          - "--configdb"
          - "config/zlgmongo-config-0.zlgmongo-config.zlg-zws.svc.cluster.local:27019"
          - "--bind_ip_all"
      volumes:
        - name: keyfile
          secret:
              secretName: zlgmongo-keyfile
              defaultMode: 0600 
              items:
                - key: keyfile.key
                  path: keyfile.key
---
apiVersion: v1
kind: Service
metadata:
  name: zlgmongo
  namespace: zlg-zws
spec:
  selector:
    app: zlgmongo
  ports:
  - port: 27017
    targetPort: 27017
    name: "27017"

初始化mongo config server 副本集

kubectl exec -it zlgmongo-config-0 -n zlg-zws -- bash
mongo --host zlgmongo-config-0.zlgmongo-config.zlg-zws.svc.cluster.local --port 27019
rs.initiate(
  {
    _id: "config",
    configsvr: true,
    members: [
      { _id : 0, host : "zlgmongo-config-0.zlgmongo-config.zlg-zws.svc.cluster.local:27019" }
    ]
  }
)

初始化mongo shard 副本集

kubectl exec -it zlgmongo-shard-0-0 -n zlg-zws -- bash
mongo --port 27018
rs.initiate(
  {
    _id: "zlgmongo-shard-0-0",
    members: [
      { _id : 0, host : "zlgmongo-shard-0-0.zlgmongo-shard-0.zlg-zws.svc.cluster.local:27018" }
    ]
  }
)
exit
exit
kubectl exec -it zlgmongo-shard-1-0 -n zlg-zws -- bash
mongo --port 27018
rs.initiate(
  {
    _id: "zlgmongo-shard-1-0",
    members: [
      { _id : 0, host : "zlgmongo-shard-1-0.zlgmongo-shard-1.zlg-zws.svc.cluster.local:27018" }
    ]
  }
)
exit
exit
kubectl exec -it zlgmongo-shard-2-0 -n zlg-zws -- bash
mongo --port 27018
rs.initiate(
  {
    _id: "zlgmongo-shard-2-0",
    members: [
      { _id : 0, host : "zlgmongo-shard-2-0.zlgmongo-shard-2.zlg-zws.svc.cluster.local:27018" }
    ]
  }
)

添加shard

kubectl exec -it `kubectl get po -l app=zlgmongo -n zlg-zws | grep zlgmongo | awk '{print $1}'` -n zlg-zws -- bash
mongo
sh.addShard("zlgmongo-shard-0-0/zlgmongo-shard-0-0.zlgmongo-shard-0.zlg-zws.svc.cluster.local:27018")
sh.addShard("zlgmongo-shard-1-0/zlgmongo-shard-1-0.zlgmongo-shard-1.zlg-zws.svc.cluster.local:27018")
sh.addShard("zlgmongo-shard-2-0/zlgmongo-shard-2-0.zlgmongo-shard-2.zlg-zws.svc.cluster.local:27018")