27.链路追踪-skywalking-oap

27.链路追踪-skywalking-oap

我们这里手动构造资源文件部署skywalking到k8s集群中

Helm安装参考https://github.com/apache/skywalking-kubernetes/tree/master/chart/skywalking

namespace:skywalking-ns

一、skywalking-oap资源说明

1. elasticsearch-svc.yaml
2. elasticsearch-statefulset.yaml
3. oap-rbac.yaml
4. oap-svc.yaml
5. oap-es-init-job.yaml
6. oap-deployment.yaml
7. ui-svc.yaml
8. ui-deployment.yaml

二、skywalking-oap资源

1. elasticsearch

用于存储skywalking oap接收到的数据

• elasticsearch-svc.yaml

kind: Service
apiVersion: v1
metadata:
  namespace: skywalking-ns
  name: elasticsearch-master
  labels:
    heritage: "Helm"
    release: "hzw-od-sw"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  annotations:
    {}
spec:
  type: ClusterIP
  selector:
    heritage: "Helm"
    release: "hzw-od-sw"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  ports:
  - name: http
    protocol: TCP
    port: 9200
  - name: transport
    protocol: TCP
    port: 9300
---
kind: Service
apiVersion: v1
metadata:
  namespace: skywalking-ns
  name: elasticsearch-master-headless
  labels:
    heritage: "Helm"
    release: "hzw-od-sw"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve
  # Create endpoints also if the related pod isn't ready
  publishNotReadyAddresses: true
  selector:
    app: "elasticsearch-master"
  ports:
  - name: http
    port: 9200
  - name: transport
    port: 9300

• elasticsearch-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  namespace: skywalking-ns
  name: elasticsearch-master
  labels:
    heritage: "Helm"
    release: "hzw-od-sw"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  annotations:
    esMajorVersion: "7"
spec:
  serviceName: elasticsearch-master-headless
  selector:
    matchLabels:
      app: "elasticsearch-master"
  replicas: 2
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: "elasticsearch-master"
      labels:
        heritage: "Helm"
        release: "hzw-od-sw"
        chart: "elasticsearch"
        app: "elasticsearch-master"
      annotations:       
    spec:
      securityContext:
        fsGroup: 1000
        runAsUser: 1000
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - "elasticsearch-master"
            topologyKey: kubernetes.io/hostname
      terminationGracePeriodSeconds: 120
      volumes:
      initContainers:
      - name: configure-sysctl
        securityContext:
          runAsUser: 0
          privileged: true
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.5.1"
        imagePullPolicy: "IfNotPresent"
        command: ["sysctl", "-w", "vm.max_map_count=262144"]
        resources:
          {}
      containers:
      - name: "elasticsearch"
        securityContext:
          capabilities:
            drop:
            - ALL
          runAsNonRoot: true
          runAsUser: 1000
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.5.1"
        imagePullPolicy: "IfNotPresent"
        readinessProbe:
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 3
          timeoutSeconds: 5
          exec:
            command:
              - sh
              - -c
              - |
                #!/usr/bin/env bash -e
                # If the node is starting up wait for the cluster to be ready (request params: 'wait_for_status=green&timeout=1s' )
                # Once it has started only check that the node itself is responding
                START_FILE=/tmp/.es_start_file
                http () {
                    local path="${1}"
                    if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then
                      BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}"
                    else
                      BASIC_AUTH=''
                    fi
                    curl -XGET -s -k --fail ${BASIC_AUTH} http://127.0.0.1:9200${path}
                }
                if [ -f "${START_FILE}" ]; then
                    echo 'Elasticsearch is already running, lets check the node is healthy and there are master nodes available'
                    http "/_cluster/health?timeout=0s"
                else
                    echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )'
                    if http "/_cluster/health?wait_for_status=green&timeout=1s" ; then
                        touch ${START_FILE}
                        exit 0
                    else
                        echo 'Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )'
                        exit 1
                    fi
                fi
        ports:
        - name: http
          containerPort: 9200
        - name: transport
          containerPort: 9300
        resources:
          limits:
            cpu: 1000m
            memory: 2Gi
          requests:
            cpu: 100m
            memory: 2Gi
        env:
          - name: node.name
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: cluster.initial_master_nodes
            value: "elasticsearch-master-0,elasticsearch-master-1,"    # es集群节点名，按节点数量
            # value: "elasticsearch-master-0,elasticsearch-master-1,elasticsearch-master-2,"
          - name: discovery.seed_hosts
            value: "elasticsearch-master-headless"
          - name: cluster.name
            value: "elasticsearch"
          - name: network.host
            value: "0.0.0.0"
          - name: ES_JAVA_OPTS
            value: "-Xmx1g -Xms1g"
          - name: node.data
            value: "true"
          - name: node.ingest
            value: "true"
          - name: node.master
            value: "true"
        volumeMounts:

因为elasticsearch是有状态服务，我们使用StatefulSet资源来部署elasticsearch

2. oap-server

• oap-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: hzw-od-sw
    chart: skywalking-4.0.0
    component: "oap"
    heritage: Helm
    release: hzw-od-sw
  name: hzw-od-sw-skywalking-oap
  namespace: skywalking-ns
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: skywalking-ns
  name: hzw-od-sw-skywalking
  labels:
    app: hzw-od-sw
    chart: "skywalking-4.0.0"
    release: "hzw-od-sw"
    heritage: "Helm"
rules:
  - apiGroups: [""]
    resources: ["pods","configmaps"]
    verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  namespace: skywalking-ns
  name: hzw-od-sw-skywalking
  labels:
    app: hzw-od-sw
    chart: "skywalking-4.0.0"
    release: "hzw-od-sw"
    heritage: "Helm"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: hzw-od-sw-skywalking
subjects:
  - kind: ServiceAccount
    name: hzw-od-sw-skywalking-oap
    namespace: skywalking-ns

• oap-svc.yaml

---
apiVersion: v1
kind: Service
metadata:
  namespace: skywalking-ns
  name: hzw-od-sw-skywalking-oap
  labels:
    app: hzw-od-sw
    chart: skywalking-4.0.0
    component: "oap"
    heritage: Helm
    release: hzw-od-sw
spec:
  type: ClusterIP
  ports:
  - port: 12800
    name: rest
  - port: 11800
    name: grpc
  selector:
    app: hzw-od-sw
    component: "oap"
    release: hzw-od-sw
---

• oap-es-init-job.yaml

apiVersion: batch/v1
kind: Job
metadata:
  namespace: skywalking-ns
  name: "hzw-od-sw-es-init"
  labels:
    app: hzw-od-sw
    chart: skywalking-4.0.0
    component: "hzw-od-sw-skywalking-job"
    heritage: Helm
    release: hzw-od-sw
spec:
  template:
    metadata:
      name: "hzw-od-sw-es-init"
      labels:
        app: hzw-od-sw
        component: "hzw-od-sw-skywalking-job"
        release: hzw-od-sw
    spec:
      restartPolicy: Never
      initContainers:
      - name: wait-for-elasticsearch
        image: busybox:1.30
        imagePullPolicy: IfNotPresent
        command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 elasticsearch-master 9200 && exit 0 || sleep 5; done; exit 1']
      containers:
      - name: oap
        image: apache/skywalking-oap-server:8.1.0-es7
        imagePullPolicy: IfNotPresent
        env:
        - name: JAVA_OPTS
          value: "-Xmx2g -Xms2g -Dmode=init"
        - name: SW_STORAGE
          value: elasticsearch7
        - name: SW_STORAGE_ES_CLUSTER_NODES
          value: "elasticsearch-master:9200"
---

在elasticsearch中初始化skywalking的数据表等资源

• oap-deployment.yaml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: hzw-od-sw
    chart: skywalking-4.0.0
    component: "oap"
    heritage: Helm
    release: hzw-od-sw
  name: hzw-od-sw-skywalking-oap
  namespace: skywalking-ns
spec:
  replicas: 2
  selector:
    matchLabels:
      app: hzw-od-sw
      component: "oap"
      release: hzw-od-sw
  template:
    metadata:
      labels:
        app: hzw-od-sw
        component: "oap"
        release: hzw-od-sw
    spec:
      serviceAccountName: hzw-od-sw-skywalking-oap
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            podAffinityTerm:
              topologyKey: kubernetes.io/hostname
              labelSelector:
                matchLabels:
                  app: "hzw-od-sw"
                  release: "hzw-od-sw"
                  component: "oap"
      initContainers:
      - name: wait-for-elasticsearch
        image: busybox:1.30
        imagePullPolicy: IfNotPresent
        command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 elasticsearch-master 9200 && exit 0 || sleep 5; done; exit 1']
      containers:
      - name: oap
        image: apache/skywalking-oap-server:8.1.0-es7
        imagePullPolicy: IfNotPresent
        livenessProbe:
          tcpSocket:
            port: 12800
          initialDelaySeconds: 15
          periodSeconds: 20
        readinessProbe:
          tcpSocket:
            port: 12800
          initialDelaySeconds: 15
          periodSeconds: 20
        ports:
        - containerPort: 11800
          name: grpc
        - containerPort: 12800
          name: rest
        env:
        - name: JAVA_OPTS
          value: "-Dmode=no-init -Xmx2g -Xms2g"
        - name: SW_CLUSTER
          value: kubernetes
        - name: SW_CLUSTER_K8S_NAMESPACE
          value: "skywalking-ns"
        - name: SW_CLUSTER_K8S_LABEL
          value: "app=hzw-od-sw,release=hzw-od-sw,component=oap"
        - name: SKYWALKING_COLLECTOR_UID
          valueFrom:
            fieldRef:
              fieldPath: metadata.uid
        - name: SW_STORAGE
          value: elasticsearch7
        - name: SW_STORAGE_ES_CLUSTER_NODES
          value: "elasticsearch-master:9200"
        volumeMounts:
      volumes:
---

3. skywalking-ui

• ui-svc.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app: hzw-od-sw
    chart: skywalking-4.0.0
    component: "ui"
    heritage: Helm
    release: hzw-od-sw
  name: hzw-od-sw-skywalking-ui
  namespace: skywalking-ns
spec:
  type: ClusterIP
  ports:
    - port: 80
      targetPort: 8080
      protocol: TCP
  selector:
    app: hzw-od-sw
    component: "ui"
    release: hzw-od-sw
---

• ui-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: skywalking-ns
  name: hzw-od-sw-skywalking-ui
  labels:
    app: hzw-od-sw
    chart: skywalking-4.0.0
    component: "ui"
    heritage: Helm
    release: hzw-od-sw
spec:
  replicas: 1
  selector:
    matchLabels:
      app: hzw-od-sw
      component: "ui"
      release: hzw-od-sw
  template:
    metadata:
      labels:
        app: hzw-od-sw
        component: "ui"
        release: hzw-od-sw
    spec:
      containers:
      - name: ui
        image: apache/skywalking-ui:8.1.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
          name: page
        env:
        - name: SW_OAP_ADDRESS
          value: hzw-od-sw-skywalking-oap:12800
---