在k8s部署flink standaloneSession模式 job-manager 高可用集群搭建

最新推荐文章于 2024-07-06 20:07:09 发布

yongge

最新推荐文章于 2024-07-06 20:07:09 发布

阅读量272

点赞数 7

文章标签： kubernetes flink 容器

本文链接：https://blog.csdn.net/jy739761380/article/details/137343712

版权

1. 创建namespace及serviceaccount

创建kubernets命名空间
  kubectl create namespace app-flink
创建该空间的用户 flink (自定义)
  kubectl create serviceaccount flink -n  app-flink
给该用户赋权
kubectl create clusterrolebinding flink-role-binding-flink \
        --clusterrole=edit \
        --serviceaccount=app-flink:flink

（不创建用户和赋权启动时候会报 "权限"错）
kubectl create clusterrolebinding endpoints-reader-mydefault \
  --clusterrole=endpoints-reader  \
  --serviceaccount=app-flink:flink

2. 在k8s上部署minio

① 创建minio-pvc.yml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-minio
  namespace: app-flink
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 40Gi

执行命令：kubectl create -f minio-pvc.yml

② 创建minio的deployment及service(minio.yml)

# 添加
apiVersion: v1
kind: Service
metadata:
  name: minio-svc
  namespace: app-flink
  labels:
    app: minio
spec:
  type: NodePort
  ports:
    - name: minio-port
      protocol: TCP
      port: 9000
      nodePort: 32600
      targetPort: 9000
    - name: minio-http
        protocol: TCP
        port: 9009
        nodePort: 32601
        targetPort: 9009
  selector:
    app: minio
---
apiVersion: v1
kind: Pod
metadata:
  name: minio
  namespace: app-flink
  labels:
    app: minio
spec:
  containers:
    - name: minio
      env:
        - name: MINIO_ACCESS_KEY
          value: "admin"
        - name: MINIO_SECRET_KEY
          value: "admin123"
      image: minio/minio:latest
      args: ["server", "/data/minio", "--console-address=0.0.0.0:9009"]
      ports:
        - name: minio
          containerPort: 9000
        - name: http
          containerPort: 9009
      volumeMounts:
        - name: data
          mountPath: /data/minio
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: pvc-minio

在对应的节点上创建目录：mkdir -p /data/minio

执行命令：kubectl create -f minio.yml

3.创建flink配置文件（flink-configuration-configmap.yml)

apiVersion: v1
kind: ConfigMap
metadata:
  name: flink-session-config
  namespace: app-flink
  labels:
    app: flink
data:
  flink-conf.yaml: |+
    kubernetes.cluster-id: app-flink-session
    # 所在的命名空间
    kubernetes.namespace: app-flink
    jobmanager.rpc.address: flink-jobmanager
    taskmanager.numberOfTaskSlots: 16
    blob.server.port: 6124
    jobmanager.rpc.port: 6123
    taskmanager.rpc.port: 6122
    queryable-state.proxy.ports: 6125
    jobmanager.memory.process.size: 1600m
    taskmanager.memory.process.size: 2867m
    parallelism.default: 2
    execution.checkpointing.interval: 10s    
    # 文件系统
    fs.default-scheme: s3
    # s3地址
    s3.endpoint: https://app-flink.s3.cn-north-1.amazonaws.com.cn/
    # s3的bucket
    s3.flink.bucket: app-flink
    s3.access-key: AKIA2MCJLQSFKHMFQ47Q
    s3.secret-key: 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
    # 状态存储格式
    state.backend: rocksdb
    s3.path.style.access: true
    blob.storage.directory: /opt/flink/tmp/blob
    web.upload.dir: /opt/flink/tmp/upload
    io.tmp.dirs: /opt/flink/tmp
    # 状态管理
    # checkpoint存储地址
    state.checkpoints.dir: s3://app-flink/state/checkpoint
    # savepoint存储地址
    state.savepoints.dir: s3://app-flink/state/savepoint
    # checkpoint间隔
    execution.checkpointing.interval: 5000
    execution.checkpointing.mode: EXACTLY_ONCE
    # checkpoint保留数量
    state.checkpoints.num-retained: 3
    # history-server# 监视以下目录中已完成的作业
    jobmanager.archive.fs.dir: s3://app-flink/completed-jobs
    # 每 10 秒刷新一次
    historyserver.archive.fs.refresh-interval: 10000
    historyserver.archive.fs.dir: s3://app-flink/completed-jobs
    # 高可用
    high-availability.type: kubernetes
    high-availability: org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory
    high-availability.storageDir: s3://app-flink/ha
    # 每6个小时触发一次savepoint
    kubernetes.operator.periodic.savepoint.interval: 6h
    kubernetes.operator.savepoint.history.max.age: 24h
    kubernetes.operator.savepoint.history.max.count: 5
    # Restart of unhealthy job deployments
    kubernetes.operator.cluster.health-check.enabled: true
    # Restart failed job deployments 
    kubernetes.operator.job.restart.failed: true
  log4j-console.properties: |+
    # This affects logging for both user code and Flink
    rootLogger.level = INFO
    rootLogger.appenderRef.console.ref = ConsoleAppender
    rootLogger.appenderRef.rolling.ref = RollingFileAppender

    # Uncomment this if you want to _only_ change Flink's logging
    #logger.flink.name = org.apache.flink
    #logger.flink.level = INFO

    # The following lines keep the log level of common libraries/connectors on
    # log level INFO. The root logger does not override this. You have to manually
    # change the log levels here.
    logger.akka.name = akka
    logger.akka.level = INFO
    logger.kafka.name= org.apache.kafka
    logger.kafka.level = INFO
    logger.hadoop.name = org.apache.hadoop
    logger.hadoop.level = INFO
    logger.zookeeper.name = org.apache.zookeeper
    logger.zookeeper.level = INFO

    # Log all infos to the console
    appender.console.name = ConsoleAppender
    appender.console.type = CONSOLE
    appender.console.layout.type = PatternLayout
    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n

    # Log all infos in the given rolling file
    appender.rolling.name = RollingFileAppender
    appender.rolling.type = RollingFile
    appender.rolling.append = false
    appender.rolling.fileName = ${sys:log.file}
    appender.rolling.filePattern = ${sys:log.file}.%i
    appender.rolling.layout.type = PatternLayout
    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
    appender.rolling.policies.type = Policies
    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
    appender.rolling.policies.size.size=100MB
    appender.rolling.strategy.type = DefaultRolloverStrategy
    appender.rolling.strategy.max = 10

    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
    logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline
    logger.netty.level = OF

4.整合flink及flink cdc构建Docker镜像

FROM flink:1.17.2-scala_2.12

# 需要用到的jar包
# flink-cdc
ADD flink-sql-connector-mysql-cdc-2.4.0.jar /opt/flink/lib/
# jdbc连接器
ADD flink-connector-jdbc-3.1.1-1.17.jar /opt/flink/lib/
# mysql驱动
ADD mysql-connector-j-8.0.31.jar /opt/flink/lib/

RUN mkdir /opt/flink/plugins/s3-fs-presto && cp -f /opt/flink/opt/flink-s3-fs-presto-1.17.2.jar /opt/flink/plugins/s3-fs-presto/

Docker镜像编译命令 docker build -t 10.10.0.203:8188/app/flink:1.17.2_scala_12 .

5.创建jobmanager-pvc.yml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: flink-tmp
  namespace: app-flink
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 40Gi

执行命令：kubectl create -f jobmanager-pvc.yml

6.创建jobmanager-deployment.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: flink-jobmanager
  namespace: app-flink
spec:
  replicas: 2 # Set the value to greater than 1 to start standby JobManagers
  selector:
    matchLabels:
      app: flink
      component: jobmanager
  template:
    metadata:
      labels:
        app: flink
        component: jobmanager
    spec:
      containers:
        - name: jobmanager
          imagePullPolicy: Always
          image: 10.10.0.203:8188/app/flink:1.17.2_scala_12
          env:
            # 注入POD的ip到容器内
            - name: POD_IP
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            # 时区
            - name: TZ
              value: Asia/Shanghai
          # The following args overwrite the value of jobmanager.rpc.address configured in the configuration config map to POD_IP.
          args: ["jobmanager", "$(POD_IP)"]
          ports:
            - containerPort: 6123
              name: rpc
            - containerPort: 6124
              name: blob-server
            - containerPort: 8081
              name: webui
          livenessProbe:
            tcpSocket:
              port: 6123
            initialDelaySeconds: 30
            periodSeconds: 60
          resources:
            requests:
              memory: "512Mi"
              cpu: "0.1"
            limits:
              memory: "4096Mi"
              cpu: "2"
          volumeMounts:
            - name: flink-ha-config-volume
              mountPath: /opt/flink/conf
            - name: tmp-dir
              mountPath: /opt/flink/tmp
          securityContext:
            runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
      serviceAccountName: flink # Service account which has the permissions to create, edit, delete ConfigMaps
      volumes:
        - name: flink-ha-config-volume
          configMap:
            name: flink-session-config
            items:
              - key: flink-conf.yaml
                path: flink-conf.yaml
              - key: log4j-console.properties
                path: log4j-console.properties
        - name: tmp-dir
          persistentVolumeClaim:
            claimName: flink-tmp

7.创建jobmanager-service-rest.yml

apiVersion: v1
kind: Service
metadata:
  name: flink-jobmanager-rest
  namespace: app-flink
spec:
  type: NodePort
  ports:
    - name: rest
      port: 8081
      targetPort: 8081
      nodePort: 30081
  selector:
    app: flink
    component: jobmanager

执行命令：kubectl create -f jobmanager-service-rest.yml

8.创建taskmanager-deployment.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: flink-taskmanager
  namespace: app-flink
spec:
  replicas: 2
  selector:
    matchLabels:
      app: flink-taskmanager
      component: taskmanager
  template:
    metadata:
      labels:
        app: flink-taskmanager
        component: taskmanager
    spec:
      containers:
        - name: taskmanager
          image: 10.10.0.203:8188/app/flink:1.17.2_scala_12
          args: ["taskmanager"]
          ports:
            - containerPort: 6122
              name: rpc
          livenessProbe:
            tcpSocket:
              port: 6122
            initialDelaySeconds: 30
            periodSeconds: 60
          volumeMounts:
            - name: flink-ha-config-volume
              mountPath: /opt/flink/conf/
          securityContext:
            runAsUser: 9999  # refers to user _flink_ from official flink image, change if necessary
      serviceAccountName: flink
      volumes:
        - name: flink-ha-config-volume
          configMap:
            name: flink-session-config
            items:
              - key: flink-conf.yaml
                path: flink-conf.yaml
              - key: log4j-console.properties
                path: log4j-console.properties

执行命令：kubectl create -f taskmanager-deployment.yml