镜像拉取
由于下载不到国外的镜像,只能使用这个笨办法了
下载国内镜像(所有节点执行)
docker pull xxx/cephcsi:v3.6.1-xxx2.8.3.1216
docker pull registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0
docker pull registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1
docker pull registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0
docker tag xxx/cephcsi:v3.6.1-csp2.8.3.1216 quay.io/cephcsi/cephcsi:v3.6.1
docker tag registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
docker tag registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
docker tag registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
下载源码
git clone https://github.com/ceph/ceph-csi.git -b release-v3.4
cd ceph-csi/deploy/cephfs/kubernetes
修改yaml文件
把文件csi-config-map.yaml修改成
---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-
[
{
"clusterID": "a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e",
"monitors": [
"172.27.16.11:6789",
"172.27.16.3:6789",
"172.27.16.7:6789"
]
}
]
metadata:
name: ceph-csi-config
创建ceph-conf.yaml
---
# This is a sample configmap that helps define a Ceph configuration as required
# by the CSI plugins.
# Sample ceph.conf available at
# https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf Detailed
# documentation is available at
# https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/
apiVersion: v1
kind: ConfigMap
data:
ceph.conf: |
[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
# adding 'fuse_big_writes = true' option by default to override this limit
# see https://github.com/ceph/ceph-csi/issues/1928
fuse_big_writes = true
# keyring is a required key and its value should be empty
keyring: |
metadata:
name: ceph-config
创建ceph-csi-encryption-kms-config.yaml
---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-
{
"vault-test": {
"encryptionKMSType": "vault",
"vaultAddress": "http://vault.default.svc.cluster.local:8200",
"vaultAuthPath": "/v1/auth/kubernetes/login",
"vaultRole": "csi-kubernetes",
"vaultBackend": "kv-v2",
"vaultDestroyKeys": "true",
"vaultPassphraseRoot": "/v1/secret",
"vaultPassphrasePath": "ceph-csi/",
"vaultCAVerify": "false"
},
"vault-tokens-test": {
"encryptionKMSType": "vaulttokens",
"vaultAddress": "http://vault.default.svc.cluster.local:8200",
"vaultBackend": "kv-v2",
"vaultBackendPath": "secret/",
"vaultTLSServerName": "vault.default.svc.cluster.local",
"vaultCAVerify": "false",
"tenantConfigName": "ceph-csi-kms-config",
"tenantTokenName": "ceph-csi-kms-token",
"tenants": {
"my-app": {
"vaultAddress": "https://vault.example.com",
"vaultCAVerify": "true"
},
"an-other-app": {
"tenantTokenName": "storage-encryption-token",
"vaultDestroyKeys": "false"
}
}
},
"vault-tenant-sa-test": {
"encryptionKMSType": "vaulttenantsa",
"vaultAddress": "http://vault.default.svc.cluster.local:8200",
"vaultBackend": "kv-v2",
"vaultBackendPath": "shared-secrets",
"vaultDestroyKeys": "false",
"vaultTLSServerName": "vault.default.svc.cluster.local",
"vaultCAVerify": "false",
"tenantConfigName": "ceph-csi-kms-config",
"tenantSAName": "ceph-csi-vault-sa",
"tenants": {
"my-app": {
"vaultAddress": "https://vault.example.com",
"vaultCAVerify": "true"
},
"an-other-app": {
"tenantSAName": "storage-encryption-sa"
}
}
},
"vault-tenant-sa-ns-test": {
"encryptionKMSType": "vaulttenantsa",
"vaultAddress": "http://vault.default.svc.cluster.local:8200",
"vaultBackend": "kv-v2",
"vaultBackendPath": "shared-secrets",
"vaultAuthNamespace": "devops",
"vaultNamespace": "devops/homepage",
"vaultTLSServerName": "vault.default.svc.cluster.local",
"vaultCAVerify": "false",
"tenantConfigName": "ceph-csi-kms-config",
"tenantSAName": "ceph-csi-vault-sa",
"tenants": {
"webservers": {
"vaultAddress": "https://vault.example.com",
"vaultAuthNamespace": "webservers",
"vaultNamespace": "webservers/homepage",
"vaultCAVerify": "true"
},
"homepage-db": {
"vaultNamespace": "devops/homepage/database",
"tenantSAName": "storage-encryption-sa"
}
}
},
"secrets-metadata-test": {
"encryptionKMSType": "metadata"
},
"user-ns-secrets-metadata-test": {
"encryptionKMSType": "metadata",
"secretName": "storage-encryption-secret",
"secretNamespace": "default"
},
"user-secrets-metadata-test": {
"encryptionKMSType": "metadata",
"secretName": "storage-encryption-secret"
},
"ibmkeyprotect-test": {
"encryptionKMSType": "ibmkeyprotect",
"secretName": "ceph-csi-kp-credentials",
"keyProtectRegionKey": "us-south-2",
"keyProtectServiceInstanceID": "7abef064-01dd-4237-9ea5-8b3890970be3"
},
"aws-sts-metadata-test": {
"encryptionKMSType": "aws-sts-metadata",
"secretName": "ceph-csi-aws-credentials"
},
"kmip-test": {
"KMS_PROVIDER": "kmip",
"KMIP_ENDPOINT": "kmip:5696",
"KMIP_SECRET_NAME": "ceph-csi-kmip-credentials",
"TLS_SERVER_NAME": "kmip.ciphertrustmanager.local",
"READ_TIMEOUT": 10,
"WRITE_TIMEOUT": 10
}
}
metadata:
name: ceph-csi-encryption-kms-config
Footer
© 2023 GitHub, Inc.
Footer navigation
Terms
Privacy
在主节点执行
kubectl taint nodes k8s-master node-role.kubernetes.io/master-
不执行这句,会导致调度到主节点的pod处于appending状态
创建pod
kubectl apply -f ceph-csi/deploy/rbd/kubernetes/
创建成功
[root@VM-16-3-centos data]# kubectl get pods
NAME READY STATUS RESTARTS AGE
csi-rbdplugin-jw8v4 3/3 Running 0 43m
csi-rbdplugin-pncb4 3/3 Running 0 43m
csi-rbdplugin-provisioner-58ff6984fd-947m2 7/7 Running 0 3m58s
csi-rbdplugin-provisioner-58ff6984fd-9mwtl 7/7 Running 0 43m
csi-rbdplugin-provisioner-58ff6984fd-mz8r8 7/7 Running 0 43m
csi-rbdplugin-wzmlp 3/3 Running 0 43m
创建csi-secret.yaml
---
apiVersion: v1
kind: Secret
metadata:
name: csi-rbd-secret
namespace: default
stringData:
userID: admin
userKey: AQDRuF1kAAAAABAAS3AdiAWbYfhVzg+EjcQqNw==
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
clusterID: a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e
pool: rbddata
imageFeatures: layering
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
csi.storage.k8s.io/controller-expand-secret-namespace: default
csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
csi.storage.k8s.io/node-stage-secret-namespace: default
csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
- discard
创建pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: csi-rbd-sc
[root@VM-16-3-centos rbd]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-ad185da7-d9de-4520-b51d-6e61223d9042 1Gi RWO Delete Bound default/rbd-pvc csi-rbd-sc 3m32s
[root@VM-16-3-centos rbd]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
rbd-pvc Bound pvc-ad185da7-d9de-4520-b51d-6e61223d9042 1Gi RWO csi-rbd-sc 30m
[root@VM-16-3-centos rbd]# rbd ls -p rbddata
csi-vol-5cd46f69-f0c3-11ed-bb18-6ef31c6b7f26
创建POD使用pvc
apiVersion: v1
kind: Pod
metadata:
name: centos
spec:
containers:
- name: mypod1
image: centos:centos8
args:
- /bin/bash
- -c
- sleep 10; touch /tmp/healthy; sleep 30000
volumeMounts:
- mountPath: "/mydata"
name: mydata
volumes:
- name: mydata
persistentVolumeClaim:
claimName: rbd-pvc
pod创建成功
[root@k8s-node2 rbd]# kubectl get pods
NAME READY STATUS RESTARTS AGE
centos 1/1 Running 0 8m7s
csi-rbdplugin-jgrsd 3/3 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-dz6lx 7/7 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-mppdr 7/7 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-tzl6j 7/7 Running 0 14m
csi-rbdplugin-rjbnd 3/3 Running 0 14m
csi-rbdplugin-sx446 3/3 Running 0 14m