k8s对接cephRBD

1. 安装RBD提供者

cat rbd-provisioner.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: rbd-provisioner
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rbd-provisioner
  namespace: kube-system
rules:
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: rbd-provisioner
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rbd-provisioner
subjects:
- kind: ServiceAccount
  name: rbd-provisioner
  namespace: kube-system

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rbd-provisioner
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["kube-dns","coredns"]
    verbs: ["list", "get"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rbd-provisioner
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: rbd-provisioner
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: rbd-provisioner
  apiGroup: rbac.authorization.k8s.io
---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rbd-provisioner
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rbd-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: rbd-provisioner
    spec:
      containers:
      - name: rbd-provisioner
        image: "registry.cn-beijing.aliyuncs.com/golaxy-kubernetes/rbd-provisioner:latest"
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/rbd
      serviceAccount: rbd-provisioner

2. ceph集群添加pool

ceph osd pool create kube 64

3. 创建k8s集群访问ceph集群的密钥

ceph auth get-or-create client.kube mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=kube' -o ceph.client.kube.keyring

4. 获取admin和kube的key

# ceph auth get-key client.admin | base64
QVFCQ3VPRmY1ODlIT0JBQVl0eDZtRkU4VVcyK1hUNWEwbHpjWlE9PQ==
# ceph auth get-key client.kube | base64
QVFDK3dlRmZEVkZkTUJBQWp1NWVqRTIzOUxmNi9kbmFJL0JzL2c9PQ==

5. 在k8s集群上创建secret

cat secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: ceph-admin-secret
  namespace: kube-system
type: "kubernetes.io/rbd"
data:
  #ceph auth get-key client.admin | base64
  key: QVFCQ3VPRmY1ODlIT0JBQVl0eDZtRkU4VVcyK1hUNWEwbHpjWlE9PQ==
---
apiVersion: v1
kind: Secret
metadata:
  name: ceph-kube-secret
  namespace: kube-system
type: "kubernetes.io/rbd"
data:
  #ceph auth get-key client.kube | base64
  key: QVFDK3dlRmZEVkZkTUJBQWp1NWVqRTIzOUxmNi9kbmFJL0JzL2c9PQ==

6. 创建storageclass

cat rbd-storage-class.yaml

apiVersion: storage.k8s.io/v1
metadata:
  name: rbd
  namespace: kube-system
provisioner: ceph.com/rbd
parameters:
  monitors: 10.170.130.79:6789, 10.170.130.80:6789, 10.170.130.138:6789
  pool: kube
  adminId: admin
  adminSecretNamespace: kube-system
  adminSecretName: ceph-admin-secret
  userId: kube
  userSecretNamespace: kube-system
  userSecretName: ceph-kube-secret
  imageFormat: "2"
  imageFeatures: layering
allowVolumeExpansion: true

7. 创建pvc

cat pvc-test.yaml

apiVersion: v1
metadata:
  name: ceph-pvc
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: rbd
  resources:
    requests:
      storage: 2Gi

查看pvc绑定情况

kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                  STORAGECLASS   REASON   AGE
pvc-aa98b70d-f8f0-4f08-b404-9d2b8f489bda   2Gi        RWO            Delete           Bound    default/ceph-pvc       rbd                     5s

8. 创建pod

cat test-nginx.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: test-nginx
  labels:
    app: nginx
spec:
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
       maxUnavailable: 1
       maxSurge: 3
  selector:
    matchLabels:
      app: nginx

  template:
    metadata:
      labels:
        app: nginx

    spec:
      containers:
      - name: nginx
        image: repository.bde.local/gitlab-runner/nginx:1.16

        volumeMounts:
        - name: nginx
          mountPath: /mnt
      volumes:
      - name: nginx
        persistentVolumeClaim:
          claimName: ceph-pvc

部署

kubectl apply -f test-nginx.yaml

进入容器中查看挂载情况

# kubectl exec -it test-nginx-66fb679874-fk7xj sh
# df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                 198.9G    125.7G     73.2G  63% /
tmpfs                    64.0M         0     64.0M   0% /dev
tmpfs                     7.8G         0      7.8G   0% /sys/fs/cgroup
/dev/rbd0                 1.9G      6.0M      1.9G   0% /mnt