CC00289.CloudKubernetes——|KuberNetes&运维.V11|——|EFLK架构.v06|部署APP+Filebeat|

yanqi_vip

已于 2022-04-02 17:04:25 修改

阅读量175

点赞数

分类专栏： cloudv017——kubernetes.v002 文章标签： docker python kubernetes 运维 java

于 2022-03-31 13:43:00 首次发布

不予转载

本文链接：https://blog.csdn.net/yanqi_vip/article/details/123874638

版权

cloudv017——kubernetes.v002 专栏收录该内容

417 篇文章 9 订阅

订阅专栏

一、部署APP+Filebeat

### --- 创建Flebeat-configmap.yaml资源文件

~~~     # FIlebeat-congimap地址：
https://github.com/dotbalo/k8s/blob/master/fklek/6.x/filebeat-configmap.yaml

### --- 创建Filebeat-confimap

[root@k8s-master01 filebeat]# vim filebeat-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeatconf
data:
  filebeat.yml: |-
    filebeat.inputs:
    - input_type: log
      paths:
#        - /data/log/*/*.log
      tail_files: true
      fields:
        pod_name: '${podName}'
        pod_ip: '${podIp}'
        pod_deploy_name: '${podDeployName}'
        pod_namespace: '${podNamespace}'
      tags: [test-filebeat]
    output.kafka:
      hosts: ["kafka:9092"]
      topic: "test-filebeat"
      codec.json:
        pretty: false
      keep_alive: 30s

### --- 注

~~~     # 注释一：将tage改为：test-filebeat
      tags: [test-filebeat]

~~~     # 注释二：将kafka的参数改成test-filebeat
    output.kafka:
      hosts: ["kafka"]
      topic: "test-filebeat"
      codec.json:
        pretty: false
      keep_alive: 30s

~~~     # 注释三：这四个字段在和ElasticSearch生成的差不多，可以在Kibana下进行缩影
            fields:
        pod_name: '${podName}'
        pod_ip: '${podIp}'
        pod_deploy_name: '${podDeployName}'
        pod_namespace: '${podNamespace}'

~~~     # 注释四：input_type是log
    - input_type: log
    
~~~     # 注释五：读取日志的路径
      paths:
#        - /data/log/*/*.log

二、创建filebeat-configmap

### --- 创建Filebeat-configmap

[root@k8s-master01 filebeat]# kubectl create -f filebeat-configmap.yaml -n public-service
configmap/filebeatconf created

### --- 查看创建的filebeat-configmap
~~~     # 注：Filebeat-confimap不一定创建在public-service，
~~~     应该和程序的容器在同一个namespace下的，

[root@k8s-master01 filebeat]# kubectl get configmap -n public-service
NAME                       DATA   AGE
filebeatconf               1      14s

三、创建APP-Filebeat-deployment.yaml：模拟日志输出APP；创建APP配置文件

### --- 创建APP配置文件

[root@k8s-master01 filebeat]# vim app-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app
  labels:
    app: app
    env: release
spec:
  selector:
    matchLabels:
      app: app
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
  # minReadySeconds: 30
  template:
    metadata:
      labels:
        app: app
    spec:
      containers:
        - name: filebeat
          image: elastic/filebeat:7.4.2
          resources:
            requests:
              memory: "100Mi"
              cpu: "10m"
            limits:
              cpu: "200m"
              memory: "300Mi"
          imagePullPolicy: Always
          env:
            - name: podIp
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            - name: podName
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: podNamespace
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: podDeployName
              value: app
            - name: TZ
              value: "Asia/Shanghai"
          securityContext:
            runAsUser: 0
          volumeMounts:
            - name: logpath
              mountPath: /data/log/app/
            - name: filebeatconf
              mountPath: /usr/share/filebeat/filebeat.yml
              subPath: usr/share/filebeat/filebeat.yml
        - name: app
          image: alpine:3.6
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: logpath
              mountPath: /home/tomcat/target/
            - name: tz-config
              mountPath: /etc/localtime
            - mountPath: /usr/share/zoneinfo/Asia/Shanghai
              name: tz-config
            - mountPath: /etc/timezone
              name: timezone
          env:
            - name: TZ
              value: "Asia/Shanghai"
            - name: LANG
              value: C.UTF-8
            - name: LC_ALL
              value: C.UTF-8
            - name: ENV
              value: k8srelease
            - name: XMS
              value: "2048m"
            - name: XMX
              value: "2048m"
            - name: MEMORY_LIMIT
              valueFrom:
                resourceFieldRef:
                  resource: requests.memory
                  divisor: 1Mi
          command:
            - sh
            - -c
            - sleep 360000
          ports:
            - containerPort: 8080
              name: tomcat
      volumes:
        - name: tz-config
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
        - hostPath:
            path: /etc/timezone
            type: ""
          name: timezone
        - name: logpath
          emptyDir: {}
        - name: filebeatconf
          configMap:
            name: filebeatconf
            items:
              - key: filebeat.yml
                path: usr/share/filebeat/filebeat.yml

### --- 注：

~~~     # 注释一：这边做了一个sleep操作，自己手动输出日志即可
          command:
            - sh
            - -c
            - sleep 360000

~~~     # 注释二：Filebeat的镜像,7.4.2需要和前面的es和logstash版本对应
          image: elastic/filebeat:7.4.2

~~~     # 注释三：
          env:
            - name: podIp
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP      // pod的IP取的是status的值
            - name: podName
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name      // podname也是取的是metadata.name的值
            - name: podNamespace
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace // namespace也是取的是metadata.namespace
            - name: podDeployName
              value: app                        // deployment的名称是不能从metadata中指定的

~~~     # 注：通过以上4个变量可以生成Kibana中的filters；若是不添加这个参数，查找日志Kenneth不好找，因为名称是不固定的。
~~~     # 注释四：应用程序
        - name: app
          image: alpine:3.6
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: logpath
              mountPath: /home/tomcat/target/
            - name: tz-config
              mountPath: /etc/localtime
            - mountPath: /usr/share/zoneinfo/Asia/Shanghai
              name: tz-config
            - mountPath: /etc/timezone
              name: timezone

~~~     # 注释五：logpath
~~~     把logpath挂载到应用程序的输出日志/home/tomcat/target/上，
~~~     然后同时挂载到Filebeat的日志收集的目录下/data/log/app/
    
        - name: logpath
          emptyDir: {}

四、创建APP-filebeat-deploy

### --- 创建APP

[root@k8s-master01 filebeat]# kubectl create -f app-deploy.yaml -n public-service
deployment.apps/app created

### --- 查看创建APP

[root@k8s-master01 filebeat]# kubectl get po -n public-service -owide
NAME                                   READY   STATUS    RESTARTS   AGE    IP               NODE           NOMINATED NODE   READINESS GATES
app-d585bb7cf-d5gpg                    2/2     Running   0          43s    172.27.14.206    k8s-node02     <none>           <none>

### --- 注：

~~~     # 注释一：若是业务应用程序APP和Filebeat不是在同一个namespace下，
~~~     假设public-service；需要把filebeat-configmap.yamlkafka的地址需要更改一下，
~~~     创建的时候指定到对应的namespace下即可

### --- 查看pod日志，是否有报错

[root@k8s-master01 filebeat]# kubectl logs -f app-d585bb7cf-d5gpg  -n public-service filebeat
2021-05-24T15:29:15.614+0800    INFO    crawler/crawler.go:72   Loading Inputs: 1
2021-05-24T15:29:15.614+0800    WARN    [cfgwarn]   input/config.go:42  DEPRECATED: input_type input config is deprecated. Use type instead. Will be removed in version: 6.0.0
2021-05-24T15:29:15.618+0800    INFO    log/input.go:152    Configured paths: [/data/log#/*/*.log]
2021-05-24T15:29:15.618+0800    INFO    input/input.go:114  Starting input of type: log; ID: 6065173283060148621 
2021-05-24T15:29:15.618+0800    INFO    crawler/crawler.go:106  Loading and starting Inputs completed. Enabled inputs: 1