一、链路追踪
1、涉及的插件elasticsearch7.0.0、skywalking-oap-server:8.9.1、skywalking-ui:8.9.1、skywalking-agent 8.7.0
2、原理
3、实现
3.1、docker-command yml文件配置
version: '3'
services:
elasticsearch:
image: elasticsearch:7.0.0
container_name: elasticsearch
restart: always
ports:
- 9200:9200
- 9300:9300
environment:
discovery.type: single-node
TZ: Asia/Shanghai
skywalking-server:
image: apache/skywalking-oap-server:8.9.1
container_name: skywalking-server
depends_on:
- elasticsearch
links:
- elasticsearch
restart: always
ports:
- 11800:11800
- 12800:12800
environment:
TZ: Asia/Shanghai
SW_STORAGE: elasticsearch
SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
volumes:
- ./skywalking/alarm-settings.yml:/skywalking/config/alarm-settings.yml
skywalking-ui:
image: apache/skywalking-ui:8.9.1
container_name: skywalking-ui
depends_on:
- skywalking-server
links:
- skywalking-server
restart: always
ports:
- 8081:8080
environment:
SW_OAP_ADDRESS: http://skywalking-server:12800
TZ: Asia/Shanghai
networks:
monitor:
driver: bridge
3.2、alarm-settings.yml 默认配置
rules:
service_resp_time_rule:
metrics-name: service_resp_time
op: ">"
threshold: 1000
period: 10
count: 3
silence-period: 5
message: Response time of service {name} is more than 1000ms in 3 minutes of last 10 minutes.
service_sla_rule:
metrics-name: service_sla
op: "<"
threshold: 8000
period: 10
count: 2
silence-period: 3
message: Successful rate of service {name} is lower than 80% in 2 minutes of last 10 minutes
service_resp_time_percentile_rule:
metrics-name: service_percentile
op: ">"
threshold: 1000,1000,1000,1000,1000
period: 10
count: 3
silence-period: 5
message: Percentile response time of service {name} alarm in 3 minutes of last 10 minutes, due to more than one condition of p50 > 1000, p75 > 1000, p90 > 1000, p95 > 1000, p99 > 1000
service_instance_resp_time_rule:
metrics-name: service_instance_resp_time
op: ">"
threshold: 1000
period: 10
count: 2
silence-period: 5
message: Response time of service instance {name} is more than 1000ms in 2 minutes of last 10 minutes
database_access_resp_time_rule:
metrics-name: database_access_resp_time
threshold: 1000
op: ">"
period: 10
count: 2
message: Response time of database access {name} is more than 1000ms in 2 minutes of last 10 minutes
endpoint_relation_resp_time_rule:
metrics-name: endpoint_relation_resp_time
threshold: 1000
op: ">"
period: 10
count: 2
message: Response time of endpoint relation {name} is more than 1000ms in 2 minutes of last 10 minutes
webhooks:
3.3、idea添加vm参数配置启动
-javaagent:H:\space\apache-skywalking-apm-bin-es7\agent\skywalking-agent.jar
-DSW_AGENT_NAME=film-server
-DSW_AGENT_COLLECTOR_BACKEND_SERVICES=192.168.1.9:11800
3.4、效果:链路、vm等查看
二、elk标准日志收集
1、涉及的技术栈:elasticsearch、logstash、kibana、kafka、zookeeper
2、原理
3、服务器实现
3.1、docker-command的yml配置
version: '3'
services:
elasticsearch:
image: elasticsearch:7.0.0
container_name: elasticsearch
restart: always
ports:
- 9200:9200
- 9300:9300
environment:
discovery.type: single-node
TZ: Asia/Shanghai
kibana:
image: kibana:7.0.0
container_name: kibana
ports:
- 5601:5601
depends_on:
- elasticsearch
links:
- elasticsearch:es
environment:
- "elasticsearch.hosts=http://es:9200"
restart: always
logstash:
image: logstash:6.4.0
container_name: logstash
volumes:
- ./logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
depends_on:
- elasticsearch
links:
- elasticsearch:es
ports:
- 4560:4560
kafka:
image: wurstmeister/kafka
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_HOST_NAME: 192.168.1.9
KAFKA_CREATE_TOPICS: "test:1:1"
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- /var/run/docker.sock:/var/run/docker.sock
ports:
- 6379:6379
zookeeper:
image: zookeeper:3.5
ports:
- 2181:2181
volumes:
- ./zookeeper/data:/zk/data
- ./zookeeper/conf:/zk/conf
networks:
monitor:
driver: bridge
3.2、logstash.conf基本配置
input { #logstash直接收集日志的tcp端口
tcp {
mode => "server"
host => "0.0.0.0"
port => 4560
codec => json_lines
}
}
output { #logstash将收集到的日志发送到es里存储
elasticsearch {
hosts => "es:9200"
index => "springboot-logstash-%{+YYYY.MM.dd}"
}
}
3.3、logstash安装插件
[root@anonymous dev]# docker exec -it logstash /bin/bash
bash-4.2$ cd /bin/
bash-4.2$ logstash-plugin install logstash-codec-json_lines
bash-4.2$ exit
[root@anonymous dev]# docker restart logstash
4、微服务配置
4.1、导入pom依赖
<!--集成logstash-->
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>5.3</version>
</dependency>
4.2、在resources新增xml配置并重启微服务
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE configuration>
<configuration>
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<!--应用名称-->
<property name="APP_NAME" value="common-ser"/>
<!--日志文件保存路径-->
<property name="LOG_FILE_PATH" value="${LOG_FILE:-${LOG_PATH:-${LOG_TEMP:-${java.io.tmpdir:-/tmp}}}/logs}"/>
<contextName>${APP_NAME}</contextName>
<!--每天记录日志到文件appender-->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_FILE_PATH}/${APP_NAME}-%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
</encoder>
</appender>
<!--输出到logstash的appender-->
<appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<!--可以访问的logstash日志收集端口-->
<destination>192.168.1.9:4560</destination>
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder"/>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE"/>
<appender-ref ref="LOGSTASH"/>
</root>
</configuration>
4.3、打开kibaba( 我的本地地址是http://192.168.1.9:5601/)配置,选择@timestamp完成
4.4、利用sentinel限流异常制造点日志
三、高并发场景日志收集升级
1、将服务器logstash.conf中的input的tcp修改为
kafka {
id => "kafka_input_9527" #随便写
bootstrap_servers => "192.168.1.9:9092"
topics => ["test"]
auto_offset_reset => "latest"
}
2、修改微服务配置
2.1、新增pom依赖
<dependency>
<groupId>com.github.danielwegener</groupId>
<artifactId>logback-kafka-appender</artifactId>
<version>0.2.0-RC2</version>
</dependency>
2.2、将resources-->xml 中 logstash的appender替换成kafka的appender
<!--输出到kafka的appender-->
<appender name="KAFKA" class="com.github.danielwegener.logback.kafka.KafkaAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
<topic>test</topic>
<!-- 不关心key的分区策略 -->
<keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy" />
<!-- 异步发送消息,不会阻塞应用程序 -->
<deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />
<producerConfig>bootstrap.servers=192.168.1.9:9092</producerConfig>
<!-- 不需要等待kafka的ack确认,对于日志数据一般都选这种,可能会丢一点日志,但是关系不大 -->
<producerConfig>acks=0</producerConfig>
<!-- 如果kafka挂了,日志会发送到console上 -->
<appender-ref ref="CONSOLE"/>
</appender>
2.3、改后的效果参考