ceph-csi rbd 容器部署

这名字很喜庆

已于 2023-05-12 21:04:05 修改

阅读量350

点赞数

文章标签： docker 容器 kubernetes

于 2023-05-12 18:03:08 首次发布

本文链接：https://blog.csdn.net/ZheMingZiHenXiQing/article/details/130647713

版权

镜像拉取

由于下载不到国外的镜像，只能使用这个笨办法了

下载国内镜像（所有节点执行）

docker pull xxx/cephcsi:v3.6.1-xxx2.8.3.1216
docker pull registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0
docker pull registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1
docker pull registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0

docker tag xxx/cephcsi:v3.6.1-csp2.8.3.1216 quay.io/cephcsi/cephcsi:v3.6.1
docker tag registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
docker tag registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
docker tag registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0

下载源码

git clone https://github.com/ceph/ceph-csi.git -b release-v3.4
cd ceph-csi/deploy/cephfs/kubernetes

修改yaml文件

把文件csi-config-map.yaml修改成

---
apiVersion: v1
kind: ConfigMap
data:
  config.json: |-
    [
      {
        "clusterID": "a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e",
        "monitors": [
          "172.27.16.11:6789",
          "172.27.16.3:6789",
          "172.27.16.7:6789"
        ]
      }
    ]
metadata:
  name: ceph-csi-config

创建ceph-conf.yaml

---
# This is a sample configmap that helps define a Ceph configuration as required
# by the CSI plugins.

# Sample ceph.conf available at
# https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf Detailed
# documentation is available at
# https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/
apiVersion: v1
kind: ConfigMap
data:
  ceph.conf: |
    [global]
      auth_cluster_required = cephx
      auth_service_required = cephx
      auth_client_required = cephx
      # Workaround for http://tracker.ceph.com/issues/23446
      fuse_set_user_groups = false

      # ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
      # adding 'fuse_big_writes = true' option by default to override this limit
      # see https://github.com/ceph/ceph-csi/issues/1928
      fuse_big_writes = true
  # keyring is a required key and its value should be empty
  keyring: |
metadata:
  name: ceph-config

创建ceph-csi-encryption-kms-config.yaml

---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-
  {
    "vault-test": {
      "encryptionKMSType": "vault",
      "vaultAddress": "http://vault.default.svc.cluster.local:8200",
      "vaultAuthPath": "/v1/auth/kubernetes/login",
      "vaultRole": "csi-kubernetes",
      "vaultBackend": "kv-v2",
      "vaultDestroyKeys": "true",
      "vaultPassphraseRoot": "/v1/secret",
      "vaultPassphrasePath": "ceph-csi/",
      "vaultCAVerify": "false"
    },
    "vault-tokens-test": {
        "encryptionKMSType": "vaulttokens",
        "vaultAddress": "http://vault.default.svc.cluster.local:8200",
        "vaultBackend": "kv-v2",
        "vaultBackendPath": "secret/",
        "vaultTLSServerName": "vault.default.svc.cluster.local",
        "vaultCAVerify": "false",
        "tenantConfigName": "ceph-csi-kms-config",
        "tenantTokenName": "ceph-csi-kms-token",
        "tenants": {
            "my-app": {
                "vaultAddress": "https://vault.example.com",
                "vaultCAVerify": "true"
            },
            "an-other-app": {
                "tenantTokenName": "storage-encryption-token",
                "vaultDestroyKeys": "false"
            }
        }
    },
    "vault-tenant-sa-test": {
        "encryptionKMSType": "vaulttenantsa",
        "vaultAddress": "http://vault.default.svc.cluster.local:8200",
        "vaultBackend": "kv-v2",
        "vaultBackendPath": "shared-secrets",
        "vaultDestroyKeys": "false",
        "vaultTLSServerName": "vault.default.svc.cluster.local",
        "vaultCAVerify": "false",
        "tenantConfigName": "ceph-csi-kms-config",
        "tenantSAName": "ceph-csi-vault-sa",
        "tenants": {
            "my-app": {
                "vaultAddress": "https://vault.example.com",
                "vaultCAVerify": "true"
            },
            "an-other-app": {
                "tenantSAName": "storage-encryption-sa"
            }
        }
    },
    "vault-tenant-sa-ns-test": {
        "encryptionKMSType": "vaulttenantsa",
        "vaultAddress": "http://vault.default.svc.cluster.local:8200",
        "vaultBackend": "kv-v2",
        "vaultBackendPath": "shared-secrets",
        "vaultAuthNamespace": "devops",
        "vaultNamespace": "devops/homepage",
        "vaultTLSServerName": "vault.default.svc.cluster.local",
        "vaultCAVerify": "false",
        "tenantConfigName": "ceph-csi-kms-config",
        "tenantSAName": "ceph-csi-vault-sa",
        "tenants": {
            "webservers": {
                "vaultAddress": "https://vault.example.com",
                "vaultAuthNamespace": "webservers",
                "vaultNamespace": "webservers/homepage",
                "vaultCAVerify": "true"
            },
            "homepage-db": {
                "vaultNamespace": "devops/homepage/database",
                "tenantSAName": "storage-encryption-sa"
            }
        }
    },
    "secrets-metadata-test": {
        "encryptionKMSType": "metadata"
    },
    "user-ns-secrets-metadata-test": {
      "encryptionKMSType": "metadata",
      "secretName": "storage-encryption-secret",
      "secretNamespace": "default"
    },
    "user-secrets-metadata-test": {
      "encryptionKMSType": "metadata",
      "secretName": "storage-encryption-secret"
    },
    "ibmkeyprotect-test": {
      "encryptionKMSType": "ibmkeyprotect",
      "secretName": "ceph-csi-kp-credentials",
      "keyProtectRegionKey": "us-south-2",
      "keyProtectServiceInstanceID": "7abef064-01dd-4237-9ea5-8b3890970be3"
    },
    "aws-sts-metadata-test": {
      "encryptionKMSType": "aws-sts-metadata",
      "secretName": "ceph-csi-aws-credentials"
    },
   "kmip-test": {
      "KMS_PROVIDER": "kmip",
      "KMIP_ENDPOINT": "kmip:5696",
      "KMIP_SECRET_NAME": "ceph-csi-kmip-credentials",
      "TLS_SERVER_NAME": "kmip.ciphertrustmanager.local",
      "READ_TIMEOUT": 10,
      "WRITE_TIMEOUT": 10
    }
  }
metadata:
name: ceph-csi-encryption-kms-config
Footer
© 2023 GitHub, Inc.
Footer navigation
Terms
Privacy

在主节点执行

kubectl taint nodes k8s-master node-role.kubernetes.io/master-

不执行这句，会导致调度到主节点的pod处于appending状态

创建pod

kubectl apply -f ceph-csi/deploy/rbd/kubernetes/

创建成功

[root@VM-16-3-centos data]# kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
csi-rbdplugin-jw8v4                          3/3     Running   0          43m
csi-rbdplugin-pncb4                          3/3     Running   0          43m
csi-rbdplugin-provisioner-58ff6984fd-947m2   7/7     Running   0          3m58s
csi-rbdplugin-provisioner-58ff6984fd-9mwtl   7/7     Running   0          43m
csi-rbdplugin-provisioner-58ff6984fd-mz8r8   7/7     Running   0          43m
csi-rbdplugin-wzmlp                          3/3     Running   0          43m

创建csi-secret.yaml

---
apiVersion: v1
kind: Secret
metadata:
  name: csi-rbd-secret
  namespace: default
stringData:
  userID: admin
  userKey: AQDRuF1kAAAAABAAS3AdiAWbYfhVzg+EjcQqNw==

---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
   clusterID: a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e
   pool: rbddata
   imageFeatures: layering
   csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
   csi.storage.k8s.io/provisioner-secret-namespace: default
   csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
   csi.storage.k8s.io/controller-expand-secret-namespace: default
   csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
   csi.storage.k8s.io/node-stage-secret-namespace: default
   csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
   - discard

创建pvc

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rbd-pvc
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: csi-rbd-sc

[root@VM-16-3-centos rbd]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS   REASON   AGE
pvc-ad185da7-d9de-4520-b51d-6e61223d9042   1Gi        RWO            Delete           Bound    default/rbd-pvc   csi-rbd-sc              3m32s
[root@VM-16-3-centos rbd]# kubectl get pvc
NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
rbd-pvc   Bound    pvc-ad185da7-d9de-4520-b51d-6e61223d9042   1Gi        RWO            csi-rbd-sc     30m
[root@VM-16-3-centos rbd]# rbd ls -p rbddata
csi-vol-5cd46f69-f0c3-11ed-bb18-6ef31c6b7f26

创建POD使用pvc

apiVersion: v1
kind: Pod
metadata:
  name: centos
spec:
  containers:
    - name: mypod1
      image: centos:centos8
      args:
      - /bin/bash
      - -c
      - sleep 10; touch /tmp/healthy; sleep 30000
      volumeMounts:
      - mountPath: "/mydata"
        name: mydata
  volumes:
    - name: mydata
      persistentVolumeClaim:
        claimName: rbd-pvc

pod创建成功

[root@k8s-node2 rbd]# kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
centos                                       1/1     Running   0          8m7s
csi-rbdplugin-jgrsd                          3/3     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-dz6lx   7/7     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-mppdr   7/7     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-tzl6j   7/7     Running   0          14m
csi-rbdplugin-rjbnd                          3/3     Running   0          14m
csi-rbdplugin-sx446                          3/3     Running   0          14m