k8s 对接ceph并使用StorageClass动态创建pvc

最新推荐文章于 2024-03-01 10:23:34 发布

隔壁老姚

最新推荐文章于 2024-03-01 10:23:34 发布

阅读量4.7k

点赞数 5

分类专栏： k8s docker

本文链接：https://blog.csdn.net/weixin_42143049/article/details/108262647

版权

k8s 同时被 2 个专栏收录

9 篇文章 1 订阅

订阅专栏

docker

4 篇文章 0 订阅

订阅专栏

k8s 对接ceph并使用StorageClass动态创建pvc

前提条件，已经安装ceph和k8s，具体环境信息如下：

一. 服务器环境

操作系统：centos 7.6
ceph版本：14.2.10（nautilus）
k8s版本：v1.18.0
arch: ARM64
GO版本：go1.14.6
docker版本：19.03.12

在k8s中使用volume主要有两种使用方式，一种的静态方法，一种是动态方法，本文主要介绍动态方法。k8s集群默认不支持动态创建pvc，官方推出了external-storage插件，我这里的k8s环境是通过容器运行的，所以本教程也主要介绍通过容器使用ceph-provisioner。

二. 使用cephfs

1.创建ceph-secret

[root@k8s-master cephfs]# ceph auth get-key client.admin|base64
QVF***************************************9PQ==
[root@k8s-master cephfs]# cat ceph-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: ceph-secret
  namespace: cephfs
data:
  key: QVF***************************************9PQ==

[root@k8s-master cephfs]# kubectl create -f ceph-secret.yaml

ceph-secret中存储的ceph admin用户的认证信息必须使用base64加密

2.创建使用cephfs-provisioner需要的相关rbac信息

2.1 创建cephfs命名空间

kubectl create ns cephfs

2.2 创建rbac

apiVersion: v1
kind: ServiceAccount
metadata:
  name: cephfs-provisioner
  namespace: cephfs
  
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-provisioner
  namespace: cephfs
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["kube-dns","coredns"]
    verbs: ["list", "get"]
    
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: cephfs-provisioner
  namespace: cephfs
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["create", "get", "delete"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-provisioner
subjects:
  - kind: ServiceAccount
    name: cephfs-provisioner
    namespace: cephfs
roleRef:
  kind: ClusterRole
  name: cephfs-provisioner
  apiGroup: rbac.authorization.k8s.io

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: cephfs-provisioner
  namespace: cephfs
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: cephfs-provisioner
subjects:
- kind: ServiceAccount
  name: cephfs-provisioner

3.制作cephfs-provisioner镜像

3.1 下载external-storage源码

git clone https://github.com/kubernetes-retired/external-storage.git -b master

需要将源码下载到GOPATH/src进行编译

3.2 编译源码

我这里是在ARM64架构下运行，所以编译的目标架构需指定为arm64,修改Makefile中的如下内容

cd $GOPATH/src/external-storage/ceph/cephfs
# vim Makefile
ifeq ($(REGISTRY),)
        REGISTRY = gebilaoyao/external_storage/  # 个人镜像仓库地址
endif

......

all build:
        CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -ldflags '-extldflags "-static"' -o cephfs-provisioner cephfs-provisioner.go

执行编译命令：

make   # 编译生成二进制文件cephfs-provisioner

3.3 编译生成镜像

修改Dockerfile内容

vim  Dockerfile

FROM centos:7

ENV CEPH_VERSION "nautilus"  # 修改ceph版本，根据环境自行修改
COPY RPM-GPG-KEY-CentOS-7 /tmp/RPM-GPG-KEY-CentOS-7  # 需先下载此文件
RUN rpm --import 'https://download.ceph.com/keys/release.asc' && \
  rpm --import /tmp/RPM-GPG-KEY-CentOS-7 && \
  rpm -Uvh https://download.ceph.com/rpm-$CEPH_VERSION/el7/noarch/ceph-release-1-1.el7.noarch.rpm && \
  yum install -y epel-release --nogpgcheck && \
  yum install -y ceph-common python-cephfs && \
  yum clean all

COPY cephfs-provisioner /usr/local/bin/cephfs-provisioner
COPY cephfs_provisioner/cephfs_provisioner.py /usr/local/bin/cephfs_provisioner
RUN chmod -v o+x /usr/local/bin/cephfs_provisioner

执行构建镜像命令make quick-container，下载安装包过程较慢，请耐心等待镜像制作过程

4. 容器运行cephfs-provisioner插件

[root@k8s-master cephfs]# cat cephfs-provisioner.yaml
piVersion: apps/v1
kind: Deployment
metadata:
  name: cephfs-provisioner
  namespace: cephfs
  labels:
    app: cephfs-provisioner
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cephfs-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: cephfs-provisioner
    spec:
      containers:
      - name: cephfs-provisioner
        image: "gebilaoyao/ceph-fs-provisioner:latest"  # 镜像为上一步编译完成的镜像，可使用docker image ls进行查看
        resources:
          limits:
            cpu: 500m
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 64Mi
        env:
        - name: PROVISIONER_NAME                # 与storageclass的provisioner参数相同
          value: ceph.com/cephfs
        - name: PROVISIONER_SECRET_NAMESPACE    # 与rbac的namespace相同
          value: cephfs
        command:
        - "/usr/local/bin/cephfs-provisioner"
        args:
        - "-id=cephfs-provisioner-1"
      serviceAccount: cephfs-provisioner
      
[root@k8s-master cephfs]# kubectl create -f cephfs-provisioner.yaml

等待cephfs-provisionerpod 创建完成。

5. 创建storageClass验证

[root@k8s-master cephfs]# cat storage-class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: cephfs-provisioner-sc
provisioner: ceph.com/cephfs
volumeBindingMode: WaitForFirstConsumer
parameters:
  monitors: 10.0.12.95:6789  # cephfs-server-ip
  adminId: admin
  adminSecretName: ceph-secret
  adminSecretNamespace: "cephfs"
  claimRoot: /pvc-volumes
[root@k8s-master cephfs]# cat cephfs-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: cephfs-pvc
  namespace: cephfs
  annotations:
    volume.beta.kubernetes.io/storage-class: "cephfs-provisioner-sc"
spec:
  # storageClassName: cephfs-provisioner-sc
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 100Mi

创建storageClass:

[root@k8s-master cephfs]# kubectl create -f storage-class.yaml

创建pvc:

[root@k8s-master cephfs]# kubectl create -f cephfs-pvc.yam

查看pvc状态：

[root@k8s-master cephrbd]# kubectl get pvc -A
NAMESPACE   NAME                     STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS            AGE
cephfs      cephfs-pvc               Bound    pvc-a620785c-1f01-4fbc-84f4-7deb5bef067a   100Mi      RWX            cephfs-provisioner-sc   19h

可以看到创建的pvc已经自动通过storageClass创建pv进行绑定，处于Bound状态

二. 使用cephrbd