k8s 部署 Doris

1. 官网提供yaml地址下载部署

链接: https://github.com/apache/doris/tree/master/docker/runtime/k8s

2. 根据要求配置环境

链接: https://doris.apache.org/zh-CN/docs/2.0/install/cluster-deployment/k8s-deploy/install-env

2.1 软件版本要求

Docker >= 1.20
Kubernetes >= 1.19
Doris >= 2.0.0
Helm（可选） >= 3.7

2.2 关闭防火墙配置：

systemctl stop firewalld
systemctl disable firewalld

2.3 禁用和关闭 swap

在部署 Doris 时，建议关闭 swap 分区。

通过以下命令可以永久关闭 swap 分区。

echo "vm.swappiness = 0">> /etc/sysctl.conf
swapoff -a && swapon -a
sysctl -p

2.4 设置系统最大打开文件句柄数

vi /etc/security/limits.conf 
* soft nofile 65536
* hard nofile 65536

2.5 修改虚拟内存区域数量

修改虚拟内存区域至少 2000000

sysctl -w vm.max_map_count=2000000

2.6 关闭透明大页

在部署 Doris 时，建议关闭透明大页。

echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag

3. 根据服务器环境，修改doris_be.yml文件。

apiVersion: v1
kind: Service
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  ports:
    - port: 9060
      name: be-port
    - port: 8040
      name: webserver-port
    - port: 9050
      name: heartbeat-port #This name should be fixed. Doris will get the port information through this name
    - port: 8060
      name: brpc-port
  clusterIP: None
  selector:
    app: doris-be-cluster1
---
apiVersion: v1
kind: Service
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  ports:
    - port: 9060
      name: be-port
    - port: 8040
      name: webserver-port
    - port: 9050
      name: heartbeat-port #This name should be fixed. Doris will get the port information through this name
    - port: 8060
      name: brpc-port
  clusterIP: None
  selector:
    app: doris-be-cluster1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: doris-be-cluster1
  labels:
    app: doris-be-cluster1
spec:
  selector:
    matchLabels:
      app: doris-be-cluster1
  serviceName: doris-be-cluster1
  replicas: 3
  template:
    metadata:
      name: doris-be-cluster1
      labels:
        app: doris-be-cluster1
    spec:
      containers:
        - name: doris-be-cluster1
          #Need to change to real mirror information
          #image: apache-doris-be:test
          # 修改点1： 镜像地址修改为真实doris镜像，可在https://hub.docker.com/r/apache/doris/tags找到需要的镜像版本
          image: apache/doris:2.0.0_alpha-be-x86_64
          imagePullPolicy: IfNotPresent
          #节点选择
      nodeSelector:
        node: middleware
          env:
            #Specify the startup type as k8s to bypass some restrictions of the official image initialization script
            - name: BUILD_TYPE
              value: "k8s"
            # 修改点2： 增加环境变量，写明FE的IP与端口
            - name: FE_MASTER_IP
              value: "doris-follower-cluster1-0.doris-follower-cluster1.doris.svc.cluster.local"
            - name: FE_MASTER_PORT
              value: "9030"
          ports:
            - containerPort: 9060
              name: be-port
            - containerPort: 8040
              name: webserver-port
            - containerPort: 9050
              name: heartbeat-port
            - containerPort: 8060
              name: brpc-port
          volumeMounts:
              #Mount the configuration file in the way of configmap
            - name: conf
              mountPath: /opt/apache-doris/be/conf
              #Ifnot mounted, when enable_profile, error will be reported when querying the data from jdbc catalog
              #Error message: error setting certificate verify locations: CAfile:/etc/pki/tls/certs/ca-bundle.crt CApath: none
            - name: sys
              mountPath: /etc/pki
              # 修改点3： 挂载存储
              subPath: pki
              readOnly: true
              # 同修改点3
            - name: sys
              mountPath: /opt/apache-doris/be/storage
              subPath: storage
      volumes:
        - name: conf
          configMap:
            name: be-conf
        - name: sys
        # 修改点4：不使用hostpath，注释掉
            #hostPath:
            #path: /etc/pki
  # 修改点5： 增加存储配置，此处使用nfs-dynamic-class或者longhorn
  volumeClaimTemplates:
  - metadata:
      name: sys
    spec:
      storageClassName: nfs-dynamic-class
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Gi 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: be-conf
data:
  be.conf: |
    PPROF_TMPDIR="$DORIS_HOME/log/"
    sys_log_level = INFO

    be_port = 9060
    webserver_port = 8040
    heartbeat_service_port = 9050
    brpc_port = 8060

    # 修改点6： 修改网段为k8s使用网段，配置数据存储路径
    priority_networks = 10.44.0.0/16
    storage_root_path = /opt/apache-doris/be/storage

4. 根据服务器环境，修改doris_follower.yml文件

apiVersion: v1
kind: Service
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  ports:
    - port: 8030
      name: http-port
    - port: 9020
      name: rpc-port
    - port: 9030
      name: query-port
    - port: 9010
      name: edit-log-port #This name should be fixed. Doris will get the port information through this name
  clusterIP: None
  selector:
    app: doris-follower-cluster1
---
apiVersion: v1
kind: Service
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  ports:
    - port: 8030
      name: http-port
    - port: 9020
      name: rpc-port
    - port: 9030
      name: query-port
    - port: 9010
      name: edit-log-port #This name should be fixed. Doris will get the port information through this name
  clusterIP: None
  selector:
    app: doris-follower-cluster1
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: doris-follower-cluster1
  labels:
    app: doris-follower-cluster1
spec:
  selector:
    matchLabels:
      app: doris-follower-cluster1
  serviceName: doris-follower-cluster1
  # 修改点1： 修改fe副本为1
  replicas: 1
  template:
    metadata:
      name: doris-follower-cluster1
      labels:
        app: doris-follower-cluster1
    spec:
      containers:
        - name: doris-follower-cluster1
          #Need to change to real mirror information
          # 修改点2： 镜像地址修改为真实doris镜像，可在https://hub.docker.com/r/apache/doris/tags找到需要的镜像版本
          image: apache/doris:2.0.0_alpha-fe-x86_64
          imagePullPolicy: IfNotPresent
          #节点选择
      nodeSelector:
        node: middleware
          env:
            # 修改点3： 增加了APP_NAMESPACE与FE_IPADDRESS环境变量
            - name: APP_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: FE_IPADDRESS
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            #Specify the startup type as k8s to bypass some restrictions of the official image initialization script
            - name: BUILD_TYPE
              value: "k8s"
            #Initialize the fe of three nodes
            - name: FE_INIT_NUMBER
            # 修改点4： 将数量改为1
              value: "1"
            #ServiceName of bakend_cn node,（if do not have bakend_cn node,do not configure this environment variable）
            # 修改点5： 不使用cn节点，注释变量CN_SERVICE  CN_STATEFULSET
            #- name: CN_SERVICE
            #  value: "doris-cn-cluster1"
            #StatefulSetName of bakend_cn node,（if do not have bakend_cn node,do not configure this environment variable)
            #- name: CN_STATEFULSET
            #  value: "doris-cn-cluster1"
            #ServiceName of bakend node,（if do not have bakend node,do not configure this environment variable）
            - name: BE_SERVICE
              value: "doris-be-cluster1"
            #StatefulSetName of bakend node,（if do not have bakend node,do not configure this environment variable)
            - name: BE_STATEFULSET
              value: "doris-be-cluster1"
            #ServiceName of follower node,（if do not have follower node,do not configure this environment variable）
            - name: FE_SERVICE
              value: "doris-follower-cluster1"
            ##StatefulSetName of follower node,（if do not have follower node,do not configure this environment variable)
            - name: FE_STATEFULSET
              value: "doris-follower-cluster1"
          ports:
            - containerPort: 8030
              name: http-port
            - containerPort: 9020
              name: rpc-port
            - containerPort: 9030
              name: query-port
            - containerPort: 9010
              name: edit-log-port
          volumeMounts:
            #Mount the configuration file in the way of configmap
            - name: conf
              mountPath: /opt/apache-doris/fe/conf
              #In order to call the api of k8s
            - name: kube
            # 使用本地配置则为/root/.kube/config
              mountPath: /root/.kube
              readOnly: true
            # 修改点6： 配置存储，用于元数据持久化
            - name: metadata
              mountPath: /opt/apache-doris/fe/doris-meta
      volumes:
        - name: conf
          configMap:
            name: follower-conf
        - name: kube
        # 修改点7： 修改为使用configMap(此处可以不修改，使用本地配置)
          #hostPath:
            #path: /root/.kube/config
          configMap:
            name: kube-conf
  # 修改点8： 增加存储配置，此处使用nfs-dynamic-class 或者 longhorn
  volumeClaimTemplates:
  - metadata:
      name: metadata
    spec:
      storageClassName: nfs-dynamic-class 
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 10Gi
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: follower-conf
data:
  fe.conf: |
    # 修改点9： 修改网段为k8s使用网段
    priority_networks = 10.44.0.0/16
    #It can automatically maintain node information by getting the number of replicas of StatefulSet, similar to alter system add/drop back
    enable_deploy_manager = k8s
    #Automatically adjust the IP of the node according to the domain name (for example, after the pod is restarted, the domain name is still doris-be-cluster1-0-doris-be-cluster1.default.svc.cluster.local, but the IP may change from 172.16.0.9 to 172.16.0.10)
    enable_fqdn_mode = true
    LOG_DIR = ${DORIS_HOME}/log
    sys_log_level = INFO
    http_port = 8030
    rpc_port = 9020
    query_port = 9030
    edit_log_port = 9010
    #Doris needs to generate the log4j configuration file according to the fe.yml configuration information, which is written in the same directory as fe.yml by default, but the config we mount is readonly, so specify this configuration to write the log4j file to another location
    custom_config_dir = /opt/apache-doris/
    #when set to false, the backend will not be dropped and remaining in DECOMMISSION state
    drop_backend_after_decommission = false
    # 修改点10： 增加元数据、java等配置
    mysql_service_nio_enabled = true
    JAVA_OPTS = "-Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
    JAVA_OPTS_FOR_JDK_9 = "-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParalle=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time"
    meta_dir = /opt/apache-doris/fe/doris-meta
    #metadata_failure_recovery = true

5. 外部访问

kind: Service
apiVersion: v1
metadata:
  name: doris-follower-loacl
  labels:
    app: doris-follower-loacl
spec:
  ports:
    - name: http
      protocol: TCP
      port: 8030
      targetPort: 8030
      nodePort: 28030
    - name: tcp
      protocol: TCP
      port: 9030
      targetPort: 9030
      nodePort: 29030
  selector:
    app: doris-follower-cluster1
  type: NodePort

6. 部署

kubectl create ns doris
kubectl apply -f doris_be.yml -n doris
kubectl apply -f doris_follower.yml -n doris
kubectl apply -f doris-svc.yaml -n doris

7. 访问及使用, 默认账号为root，无密码

# web端访问地址
http://[节点IP]:20803/login

# 使用mysql client连接地址
host: [节点IP]
port: 29030
user: root
pass: 

# 使用mysql client连接后可修改root密码
SET PASSWORD FOR 'root' = PASSWORD('your_password');