阿里云ecs自建k8s集群安装阿里的日志服务

学海无涯码代码

已于 2024-01-17 13:18:35 修改

阅读量151

点赞数

文章标签：阿里云 kubernetes 云计算

于 2023-09-19 12:38:25 首次发布

本文链接：https://blog.csdn.net/qq_33192671/article/details/133022882

版权

安装logtail-ds

apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    deprecated.daemonset.template.generation: "1"
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"apps/v1","kind":"DaemonSet","metadata":{"annotations":{},"labels":{"k8s-app":"logtail-ds"},"name":"logtail-ds","namespace":"kube-system"},"spec":{"selector":{"matchLabels":{"k8s-app":"logtail-ds"}},"template":{"metadata":{"annotations":{"checksum/config":"bd79ee0dbc7ad66c6f652130a1d9ddbf827521ad8fa7f02d4829201f536dcf65"},"labels":{"k8s-app":"logtail-ds","kubernetes.io/cluster-service":"true","version":"v1.0"}},"spec":{"containers":[{"env":[{"name":"HTTP_PROBE_PORT","value":"7953"},{"name":"ALIYUN_LOGTAIL_CONFIG","valueFrom":{"configMapKeyRef":{"key":"log-config-path","name":"alibaba-log-configuration"}}},{"name":"ALIYUN_LOGTAIL_USER_ID","valueFrom":{"configMapKeyRef":{"key":"log-ali-uid","name":"alibaba-log-configuration"}}},{"name":"ALIYUN_LOGTAIL_USER_DEFINED_ID","valueFrom":{"configMapKeyRef":{"key":"log-machine-group","name":"alibaba-log-configuration"}}},{"name":"ALICLOUD_LOG_ACCESS_KEY_ID","valueFrom":{"configMapKeyRef":{"key":"access-key-id","name":"alibaba-log-configuration"}}},{"name":"ALICLOUD_LOG_ACCESS_KEY_SECRET","valueFrom":{"configMapKeyRef":{"key":"access-key-secret","name":"alibaba-log-configuration"}}},{"name":"ALICLOUD_LOG_DOCKER_ENV_CONFIG","value":"true"},{"name":"ALICLOUD_LOG_ECS_FLAG","value":"false"},{"name":"ALICLOUD_LOG_DEFAULT_PROJECT","valueFrom":{"configMapKeyRef":{"key":"log-project","name":"alibaba-log-configuration"}}},{"name":"ALICLOUD_LOG_ENDPOINT","valueFrom":{"configMapKeyRef":{"key":"log-endpoint","name":"alibaba-log-configuration"}}},{"name":"ALICLOUD_LOG_DEFAULT_MACHINE_GROUP","valueFrom":{"configMapKeyRef":{"key":"log-machine-group","name":"alibaba-log-configuration"}}},{"name":"ALIYUN_LOG_ENV_TAGS","value":"_node_name_|_node_ip_"},{"name":"_node_name_","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"_node_ip_","valueFrom":{"fieldRef":{"fieldPath":"status.hostIP"}}},{"name":"cpu_usage_limit","valueFrom":{"configMapKeyRef":{"key":"cpu-core-limit","name":"alibaba-log-configuration"}}},{"name":"mem_usage_limit","valueFrom":{"configMapKeyRef":{"key":"mem-limit","name":"alibaba-log-configuration"}}},{"name":"max_bytes_per_sec","valueFrom":{"configMapKeyRef":{"key":"max-bytes-per-sec","name":"alibaba-log-configuration"}}},{"name":"send_request_concurrency","valueFrom":{"configMapKeyRef":{"key":"send-requests-concurrency","name":"alibaba-log-configuration"}}}],"image":"registry.cn-shenzhen.aliyuncs.com/log-service/logtail:latest","livenessProbe":{"httpGet":{"path":"/liveness","port":7953,"scheme":"HTTP"},"initialDelaySeconds":30,"periodSeconds":60},"name":"logtail","resources":{"limits":{"cpu":2,"memory":"1024Mi"},"requests":{"cpu":"100m","memory":"256Mi"}},"securityContext":{"privileged":true},"volumeMounts":[{"mountPath":"/var/run/","name":"sock"},{"mountPath":"/logtail_host","mountPropagation":"HostToContainer","name":"root","readOnly":true}]}],"dnsPolicy":"Default","hostNetwork":true,"terminationGracePeriodSeconds":30,"tolerations":[{"operator":"Exists"}],"volumes":[{"hostPath":{"path":"/var/run/"},"name":"sock"},{"hostPath":{"path":"/"},"name":"root"}]}},"updateStrategy":{"type":"RollingUpdate"}}}
  creationTimestamp: "2023-05-11T05:46:40Z"
  generation: 1
  labels:
    k8s-app: logtail-ds
  name: logtail-ds
  namespace: kube-system
  resourceVersion: "47344878"
  uid: 41b1f4b0-95ee-46ce-b92d-85c13ca039df
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: logtail-ds
  template:
    metadata:
      annotations:
        checksum/config: bd79ee0dbc7ad66c6f652130a1d9ddbf827521ad8fa7f02d4829201f536dcf65
      creationTimestamp: null
      labels:
        k8s-app: logtail-ds
        kubernetes.io/cluster-service: "true"
        version: v1.0
    spec:
      containers:
      - env:
        - name: HTTP_PROBE_PORT
          value: "7953"
        - name: ALIYUN_LOGTAIL_CONFIG
          valueFrom:
            configMapKeyRef:
              key: log-config-path
              name: alibaba-log-configuration
        - name: ALIYUN_LOGTAIL_USER_ID
          valueFrom:
            configMapKeyRef:
              key: log-ali-uid
              name: alibaba-log-configuration
        - name: ALIYUN_LOGTAIL_USER_DEFINED_ID
          valueFrom:
            configMapKeyRef:
              key: log-machine-group
              name: alibaba-log-configuration
        - name: ALICLOUD_LOG_ACCESS_KEY_ID
          valueFrom:
            configMapKeyRef:
              key: access-key-id
              name: alibaba-log-configuration
        - name: ALICLOUD_LOG_ACCESS_KEY_SECRET
          valueFrom:
            configMapKeyRef:
              key: access-key-secret
              name: alibaba-log-configuration
        - name: ALICLOUD_LOG_DOCKER_ENV_CONFIG
          value: "true"
        - name: ALICLOUD_LOG_ECS_FLAG
          value: "false"
        - name: ALICLOUD_LOG_DEFAULT_PROJECT
          valueFrom:
            configMapKeyRef:
              key: log-project
              name: alibaba-log-configuration
        - name: ALICLOUD_LOG_ENDPOINT
          valueFrom:
            configMapKeyRef:
              key: log-endpoint
              name: alibaba-log-configuration
        - name: ALICLOUD_LOG_DEFAULT_MACHINE_GROUP
          valueFrom:
            configMapKeyRef:
              key: log-machine-group
              name: alibaba-log-configuration
        - name: ALIYUN_LOG_ENV_TAGS
          value: _node_name_|_node_ip_
        - name: _node_name_
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: _node_ip_
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.hostIP
        - name: cpu_usage_limit
          valueFrom:
            configMapKeyRef:
              key: cpu-core-limit
              name: alibaba-log-configuration
        - name: mem_usage_limit
          valueFrom:
            configMapKeyRef:
              key: mem-limit
              name: alibaba-log-configuration
        - name: max_bytes_per_sec
          valueFrom:
            configMapKeyRef:
              key: max-bytes-per-sec
              name: alibaba-log-configuration
        - name: send_request_concurrency
          valueFrom:
            configMapKeyRef:
              key: send-requests-concurrency
              name: alibaba-log-configuration
        image: registry.cn-shenzhen.aliyuncs.com/log-service/logtail:latest
        imagePullPolicy: Always
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /liveness
            port: 7953
            scheme: HTTP
          initialDelaySeconds: 30
          periodSeconds: 60
          successThreshold: 1
          timeoutSeconds: 1
        name: logtail
        resources:
          limits:
            cpu: "2"
            memory: 1Gi
          requests:
            cpu: 100m
            memory: 256Mi
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/run/
          name: sock
        - mountPath: /logtail_host
          mountPropagation: HostToContainer
          name: root
          readOnly: true
      dnsPolicy: Default
      hostNetwork: true
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      tolerations:
      - operator: Exists
      volumes:
      - hostPath:
          path: /var/run/
          type: ""
        name: sock
      - hostPath:
          path: /
          type: ""
        name: root
  updateStrategy:
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
    type: RollingUpdate
status:
  currentNumberScheduled: 2
  desiredNumberScheduled: 2
  numberAvailable: 2
  numberMisscheduled: 0
  numberReady: 2
  observedGeneration: 1
  updatedNumberScheduled: 2

安装完成后会创建 logtail-ds pod 在这里插入图片描述

创建configmap

apiVersion: v1
data:
  access-key-id: 这里要填自己的access-key-id
  access-key-secret: 这里要填自己的 access-key-secret
  cpu-core-limit: "2"
  log-ali-uid: "1833061227926848"
  log-config-path: /etc/ilogtail/conf/cn-shenzhen_internet/ilogtail_config.json
  log-endpoint: cn-shenzhen.log.aliyuncs.com
  log-machine-group: k8s-group-custom-zywl-uat
  log-project: k8s-log-custom-zywl-uat # 项目名称
  max-bytes-per-sec: "20971520"
  mem-limit: "1024"
  send-requests-concurrency: "20"
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"access-key-id":"这里要填自己的access-key-id","access-key-secret":"这里要填自己的 access-key-secret","cpu-core-limit":"2","log-ali-uid":"1833061227926848","log-config-path":"/etc/ilogtail/conf/cn-shenzhen_internet/ilogtail_config.json","log-endpoint":"cn-shenzhen.log.aliyuncs.com","log-machine-group":"k8s-group-custom-zywl-uat","log-project":"k8s-log-custom-zywl-uat","max-bytes-per-sec":"20971520","mem-limit":"1024","send-requests-concurrency":"20"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"alibaba-log-configuration","namespace":"kube-system"}}
  creationTimestamp: "2023-05-11T05:46:40Z"
  name: alibaba-log-configuration
  namespace: kube-system
  resourceVersion: "32052415"
  uid: 115023d9-b6d6-46e0-9693-0171e8c0bbbd




k8s-log-custom-zywl-uat 这个换成自己的项目名称，稍后会说下这个是哪来的

k8s-log-custom-zywl-uat 说明
在这里插入图片描述
没有的话，就需要自己去创建了

在这里插入图片描述
这里给个参考，就是项目名称

alicloud-log-k8s-custom-install.sh


#!/bin/bash

ROOTDIR=$(cd $(dirname $0) && pwd)

# Check whether the variables in the template exist in the configuration file
function checkVariableWithDefaultValue() {
  path=$1
  templates=$path/templates
  values=$path/values.yaml

  if [ ! -d $templates ]; then
    echo "[FAIL] Cannot find template files for LogService's components install"
    exit 1
  fi

  if [ ! -f $values ]; then
    echo "[FAIL] Cannot read config file for LogService's components install"
    exit 1
  fi
  template_vars=$(mktemp temp.XXXXXX)
  values_vars=$(mktemp temp.XXXXXX)
  tmp=$(mktemp temp.XXXXXX)
  for file in `ls $templates`
  do
        cat $templates/$file |tr ':' '\n'| grep -E ".*\\$\\{.*\\}.*"| sed 's/.*${\(.*\)}.*/\1/g' >> $tmp
  done
  cat $tmp|grep -v SNIPPET | uniq| sort >> $template_vars
  cat $values | grep -v -E "^#.*"|grep -vE "^$"| cut -d : -f 1| sort >> $values_vars

  grep -vf $values_vars $template_vars >  $tmp
  res=0
  if [ -s $tmp ]; then
    echo "[FAIL] The following variables cannot be found in the config file:"
    cat $tmp
    res=1
  fi

  rm -rf $tmp
  rm -rf $template_vars
  rm -rf $values_vars
  if [ $res = 1 ]; then
    echo "[FAIL] Some variables of templates are not exist in config file"
    exit 1
  fi
}

# Export the configuration file with KV pairs
function exportValues() {
  values=$1/values.yaml
  tmp=$(mktemp temp.XXXXXX)
  cat $values  | grep -v -E "^#.*"|grep -vE "^$"| while read line; do
      key=`echo $line|awk -F ":" '{print $1}'| awk '$1=$1'`
      val=`echo $line|awk -F ":" '{print $2}'| awk '$1=$1'`
      echo "export $key=$val" >> "$tmp"
  done
  source ./"$tmp"
  rm -rf "$tmp"
}

# Override export values with the input params
function overrideValues() {
  export SLS_PROJECT_NAME="k8s-log-custom-$1"
  export SLS_MACHINE_GROUP="k8s-group-custom-$1"

  export SLS_REGION="$2_internet"
  export SLS_ENDPOINT="$2.log.aliyuncs.com"
  export SLS_CONTROLLER_IAMGE="registry.$2.aliyuncs.com/log-service/alibabacloud-log-controller"
  export SLS_LOGTAIL_IAMGE="registry.$2.aliyuncs.com/log-service/logtail"

  export SLS_ALIUID="$3"
  export SLS_ACCESSKEY_ID="$4"
  export SLS_ACCESSKEY_SECRET="$5"

  export SLS_ACK_CLUSTER=false
  export SLS_LOGTAIL_CONTAINER_PRIVILEGED=true
}

# Generate the kubernetes resource yaml
function generateKubernetesResourceYaml() {
  templatePath=$1/templates
  deployPath=$2
  rm -rf "$deployPath"
  mkdir "$deployPath"
  tmp=$(mktemp temp.XXXXXX)

  if [ "${SLS_ACK_CLUSTER}" = "false" ]; then
     envsubst <"$templatePath"/"snippet_alicluster_controller"> "$tmp"
     content=$(awk '{print "          "$0}' "$tmp")
     export SNIPPET_SLS_ACK_CLUSTER_CONTROLLER="$content"

     envsubst <"$templatePath"/"snippet_alicluster_daemonset"> "$tmp"
     content=$(awk '{print "          "$0}' "$tmp")
     export SNIPPET_SLS_ACK_CLUSTER_DAEMONSET="$content"
  fi

  if [ "$SLS_CONTAINERD_USED" = "true" ]; then
     envsubst <"$templatePath"/"snippet_containerd"> "$tmp"
     content=$(awk '{print "          "$0}' "$tmp")
     export SNIPPET_SLS_CONTAINERD_USED="$content"
  fi

  if [ "$SLS_DOCKER_ENVCONFIG" = "true" ]; then
     envsubst <"$templatePath"/"snippet_dockerenvconfig"> "$tmp"
     content=$(awk '{print "          "$0}' "$tmp")
     export SNIPPET_SLS_DOCKER_ENVCONFIG="$content"
  fi

   envsubst <"$templatePath"/alicloud-log-config.yaml>  "$deployPath"/alicloud-log-config.yaml
   hash=$(sha256sum "$deployPath"/alicloud-log-config.yaml |awk -F " " '{print $1}')
   export SNIPPET_CONFIGMAP_SHA256="$hash"
   envsubst <"$templatePath"/alicloud-log-crd.yaml>  "$deployPath"/alicloud-log-crd.yaml
   envsubst <"$templatePath"/logtail-daemonset.yaml>  "$deployPath"/logtail-daemonset.yaml
   rm -rf "$tmp"
   echo "The Logtail kubernetes deploy YAML has been generated in $deployPath"
}


kubectl version
if [ $? != 0 ]; then
  echo "[ENV] Kubectl has not been installed, please install it before installing Logtail components"
  exit 1
fi

if [ $# -le 4 ] ; then
    echo "[Invalid Param], use sudo ./install-k8s-log.sh {your-project-suffix} {regionId} {aliuid} {accessKeyId} {accessKeySecret}"
    exit 1
fi

project=$1
regionId=$2
aliuid=$3
accessKeyId=$4
accessKeySecret=$5

if [ ! -d alibaba-cloud-log ]; then
  echo "Fetching the latest logserver configuration..."
  packageUrl="http://logtail-release-$regionId.oss-$regionId.aliyuncs.com/kubernetes/alibaba-cloud-log.tgz"
  wget $packageUrl -O alibaba-cloud-log.tgz
  if [ $? != 0 ]; then
      echo "[FAIL] download alibaba-cloud-log.tgz from $packageUrl failed"
      exit 1
  fi
  tar -xvf alibaba-cloud-log.tgz
fi


checkVariableWithDefaultValue "$ROOTDIR"/alibaba-cloud-log
exportValues "$ROOTDIR"/alibaba-cloud-log
overrideValues "$project" "$regionId" "$aliuid" "$accessKeyId" "$accessKeySecret"
generateKubernetesResourceYaml "$ROOTDIR"/alibaba-cloud-log "$ROOTDIR"/deploy
kubectl apply -f "$ROOTDIR"/deploy

cube_json.yml

apiVersion: log.alibabacloud.com/v1alpha1
kind: AliyunLogConfig
metadata:
  # 设置资源名，在当前Kubernetes集群内唯一。
  name: json-zywl
spec:
  # 设置Logstore名称。如果您所指定的Logstore不存在，日志服务会自动创建。
  logstore: zywl
  logtailConfig:
    # 设置采集的数据源类型。采集文本日志时，需设置为file。
    inputType: file
    # 设置Logtail配置的名称，必须与资源名（metadata.name）相同。
    configName: json-zywl
    inputDetail:
      # 指定通过JSON模式采集日志。
      logType: json_log
      # 设置日志文件的路径。
      logPath: /data/log/app
      # 设置日志文件的名称。支持通配符星号（*）和半角问号（?），例如log_*.log。
      filePattern: run.log
      # 设置时间字段，如果无指定需求，则设置为timeKey: ''。
      # 设置时间格式。如果无指定需求，则设置为timeFormat: ''。
      # 采集容器的文本日志时，需设置dockerFile为true。
      dockerFile: true
      # 设置环境变量白名单。只采集环境变量中包含EnvKey为ALIYUN_LOGTAIL_USER_DEFINED_ID的容器的文本日志。
      discardUnmatch: false

安装完成后会
在这里插入图片描述
安装完成后会有这三个pod

参考阿里的安装教程
https://help.aliyun.com/zh/sls/user-guide/install-the-logtail-component-self-built-kubernetes-cluster?spm=a2c4g.11186623.0.0.14e13285cV8ffX

在这里插入图片描述
阿里云后台配置参考

学海无涯码代码

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
阿里云ecs自建k8s集群安装阿里的日志服务

k8s-log-custom-zywl-uat 说明。安装完成后会创建 logtail-ds pod。没有的话，就需要自己去创建了。这里给个参考，就是项目名称。安装完成后会有这三个pod。安装logtail-ds。创建configmap。
复制链接

扫一扫