k8s-部署对象存储minio

环境信息

minio版本 :最新

k8s 版本1.22

使用nfs作为共享存储

一.单节点安装包部署

脚本部署，一键部署，单节点应用于数据量小，一些缓存存储，比如gitlab-runner的产物数据，maven的打包依赖数据

#!/bin/bash

# 步骤一：创建目录
sudo mkdir -p /opt/minio/{bin,conf,data}

# 步骤二：下载服务
cd /opt/minio/bin
sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio
sudo chmod +x minio

# 步骤三：创建配置文件
sudo tee /opt/minio/conf/minio.conf > /dev/null <<EOF
MINIO_VOLUMES="/opt/minio/data"
#--address指定S3 API端口，--console-address指定web控制台端口.
MINIO_OPTS="--address :39000 --console-address :39001" 
# Root user for the server.
MINIO_ROOT_USER=admin
# Root secret for the server.
MINIO_ROOT_PASSWORD=minio123
EOF

# 步骤四：创建启动命令
sudo tee /etc/systemd/system/minio.service > /dev/null <<EOF
[Unit]
Description=MinIO
Documentation=https://www.minio.org.cn/
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/opt/minio/bin/minio

[Service]
User=root
Group=root

EnvironmentFile=/opt/minio/conf/minio.conf
ExecStart=/opt/minio/bin/minio server \$MINIO_OPTS \$MINIO_VOLUMES
Restart=always

LimitNOFILE=65536
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target
EOF

# 步骤五：启动服务
sudo systemctl daemon-reload
sudo systemctl enable minio.service
sudo systemctl start minio.service
sudo systemctl status minio.service

二.单节点k8s部署

1.minio-deploy.yaml部署

apiVersion: apps/v1
kind: Deployment
metadata:
  name: minio
spec:
  replicas: 1
  selector:
    matchLabels:
      app: minio
  template:
    metadata:
      labels:
        app: minio
    spec:
      containers:
      - name: minio
        env:
        - name: MINIO_ROOT_USER
          value:  'root'
        - name: MINIO_ROOT_PASSWORD
          value:  'root123456'        
        image: minio/minio:latest
        imagePullPolicy: IfNotPresent
        command:
          - /bin/sh
          - -c
          - minio server /data --console-address ":5000"
        ports:
        - name: data
          containerPort: 9000
          protocol: "TCP"
        - name: console
          containerPort: 5000
          protocol: "TCP"
        volumeMounts:
        - name: minio-persistent-data
          mountPath: /data
      volumes:
        - name: minio-persistent-data
          nfs:
            server: 192.168.110.190 
            path: /data/data/minio

执行资源文件

kubectl -f apply  minio-deploy.yaml -n minio

2.minio-service.yaml 创建

apiVersion: v1
kind: Service
metadata:
  name: minio
spec:
  type: NodePort
  ports:
  - name: data
    port: 9000
    targetPort: 9000
    protocol: TCP
    nodePort: 30069
  - name: console
    port: 5000
    targetPort: 5000
    protocol: TCP
    nodePort: 30070
  selector:
    app: minio

执行资源文件

kubectl -f apply  minio-service.yaml -n minio

安装好后浏览器访问ui,登录密码root和root123456

http://集群内部任意ip:30070 

三.k8s集群部署

1.创建挂载多磁盘目录

mkdir -p /data/data/minio/pv{1..4}

2.创建sc.yaml

cat > sc.yaml << 'EOF'
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: minio-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
EOF

3.创建pv.yaml

cat > pv.yaml << 'EOF'
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-nfs-pv-1
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  nfs:
    server: 192.168.110.190
    path: /data/data/minio/pv1
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-nfs-pv-2
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  nfs:
    server: 192.168.110.190
    path: /data/data/minio/pv2
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-nfs-pv-3
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  nfs:
    server: 192.168.110.190
    path: /data/data/minio/pv3
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-nfs-pv-4
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  nfs:
    server: 192.168.110.190
    path: /data/data/minio/pv4
EOF

4.创建 SVC

cat > svc.yaml << 'EOF'
apiVersion: v1
kind: Service
metadata:
  name: minio-hs
  namespace: gitlab
  labels:
    app: minio
spec:
  clusterIP: None
  ports:
    - port: 9000
      name: data
    - port: 5000
      name: console
  selector:
    app: minio
---
apiVersion: v1
kind: Service
metadata:
  name: minio-sc
  namespace: gitlab
  labels:
    app: minio
spec:
  type: NodePort
  ports:
    - name: data
      port: 9000
      targetPort: 9000
      protocol: TCP
      nodePort: 30090
    - name: console
      port: 5000
      targetPort: 5000
      protocol: TCP
      nodePort: 30050
  selector:
    app: minio
EOF

5.创建 STS

cat > sts.yaml << 'EOF'
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: minio
  namespace: gitlab
spec:
  serviceName: "minio-hs"
  replicas: 4
  selector:
    matchLabels:
      app: minio
  template:
    metadata:
      labels:
        app: minio
    spec:
      containers:
      - name: minio
        env:
        - name: MINIO_ROOT_USER
          value: "admin"
        - name: MINIO_ROOT_PASSWORD
          value: "admin12345"
        image: minio/minio:RELEASE.2023-08-09T23-30-22Z
        imagePullPolicy: IfNotPresent
        command:
        - /bin/sh
        - -c
        - minio server --console-address ":5000" http://minio-{0...3}.minio-hs.minio.svc.cluster.local/data
        ports:
        - name: data
          containerPort: 9000
          protocol: "TCP"
        - name: console
          containerPort: 5000
          protocol: "TCP"
        volumeMounts:
        - name: data
          mountPath: /data
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 10Gi
      storageClassName: "minio-storage"
EOF

6.卸载服务

kubectl delete pod minio-mc -n gitlab

kubectl delete -f sts.yaml
kubectl get pod -n gitlab -o wide

kubectl delete -f svc.yaml
kubectl get svc -n gitlab

kubectl delete namespace minio
kubectl get namespace

kubectl get pvc -n gitlab
kubectl delete pvc -n migitlabnio --all

kubectl delete -f pv.yaml -n gitlab
kubectl get pv

kubectl delete -f sc.yaml -n gitlab
kubectl get sc

rm -rf /data/data/minio
mkdir -p /data/data/minio/pv{1..4}
tree /data/data/minio

四.注意事项

分部署部署时，注意使用的minio镜像，我是试了好多版本，才可以的，因为大多数存在minio-dns访问问题，各个几点之间通信异常