一、需求:错误日志Top10告警发送
二、需求分解
- jsoup实现登录,获取到cookie和token等用户鉴权信息
- 获取接口相应的key值
- 调用日志平台错误日志Top榜接口,查询到结果集
- 处理消息(调用企业微信机器人发送消息接口,电话预警功能)
- 加上定时任务,可以实现定时发送错误日志告警的功能
定时任务高度设置1小时(根据实际需要)执行一次,可以实现自动巡检错误日志的功能。以便及时发现生产问题
jsoup是java的爬虫框架,可以爬取网页数据,这里没有重点使用,只是做了个登录功能。后续可以专门它写一份爬虫的程序。。
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.12.1</version>
</dependency>
package com.smy.cbs.task;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.smy.cbs.client.NtfServiceClient;
import com.smy.cbs.entity.DelayedElement;
import com.smy.cbs.util.RetryUtil;
import com.smy.framework.redis.RedisLock;
import com.smy.smyx.scheduler.dto.TaskRequest;
import com.smy.smyx.scheduler.dto.TaskResponse;
import com.smy.smyx.scheduler.service.TaskService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.net.ssl.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @author youlu
* @ClassName logCenterTask
* @Date 2023/11/22 17:14
* @Version V1.0
**/
@Slf4j
@Service("logCenterTask")
public class LogCenterTask implements TaskService {
@Resource
private RetryUtil retryUtil;
@Resource
private NtfServiceClient ntfServiceClient;
@Value("${log.usename:aaaa}")
public String USER_NAME;
@Value("${log.password:bbbb}")
public String PASSWORD;
@Value("${log.content.warn.max:24000000}")
public int maxCount;//达到阈值则告警
@Value("${log.warn.count:100}")
public int warnCount;//字体标黄,打电话告警阈值
@Value("#{'${log.content.filter.content:输入要过滤的内容有英文逗号隔开}'.split(',')}")
public List<String> filterContent = Lists.newArrayList();//过滤内容
@Value("${log.content.length:240}")
public int SUB_CONTENT_LENGTH = 300;//截取报错内容字符串长度
@Value("${log.title.length:20}")
public int SUB_TITLE_LENGTH = 40;//截取报错类型字符串长度
@Value("${log.period.hour:24}")
public int PERIOD_HOUR;//24小时内的错误日志
@Value("#{'${log.search.system:cbs_core,rls_core,uts_core,adv_core}'.split(',')}")
public List<String> SEARCH_SYSTEM;//查询的系统
@Value("#{'${log.wx.url:https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=b54887d8-xxxx,https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=87140cb7-x123}'.split(',')}")
public List<String> WX_ROBOT_URL;
@Value("#{'${log.warn.phone:}'.split(',')}")
public List<String> MOBILE_PHONE;//预警手机
@Value("#{'${log.filter.phone.time:0,1,2,3,4,5,6,7,13,22,23}'.split(',')}")
public List<Integer> filterPhoneTime;//电话预警时间过滤
@Value("${log.phone.warn.lock.hour:6}")
public int phoneWarnLockHour;//小时
@Value("${log.url:https://log_center.xxxx.com/api/}")
public String LOG_CENTER_BASE_URL;
public static String X_Csrf_Token = "59e82a89-d671-462e-87be-ee6cd30cd7f8";
public static String COOKIE = "JSESSIONID=MWY1NjUxMDItYzU3Ny00Y2Y1LTk2OTYtZjUxZjQ5ZGIzZWRiLGwwMTU2MixyNmdNVkxsSHJWNW9kaDFoMTMxVGRNQ0oyc3c9";
public static String id = "Zf_0-osBnsFDyEFyLnk0";
public static final String TEMPLATE = "{\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"%s\"}}";
public static final String QUERY_TEMPLATE = "{\n" +
" \"app\": \"system\",\n" +
" \"source\": \"other\",\n" +
" \"query\": \"repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n| where level=\\\"ERROR\\\"\\n| eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n| stats count() as num by err_type\\n| sort 10 by num\\n| join type=inner err_type [\\n repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n | where level=\\\"ERROR\\\"\\n | eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n | fields + err_type, _raw\\n | dedup err_type\\n]\\n| rename _raw as 原始日志, num as 统计, err_type as 错误类型\",\n" +
" \"mode\": \"smart\",\n" +
" \"preview\": false,\n" +
" \"collectSize\": -1,\n" +
" \"timeout\": 1000,\n" +
" \"sorts\": []\n" +
"}";
public static final BlockingQueue<DelayedElement> delayQueue = new DelayQueue<>();
public static int telWarnIndex = 0;
@PostConstruct
public void init() {
new Thread(() -> {
while (true) {
String system = null;
try {
DelayedElement element = delayQueue.take();
String data = element.getData();
if (StringUtils.isBlank(data)) {
continue;
}
Date date = new Date();
int hour = DateUtil.hour(date, true);
if (filterPhoneTime.stream().anyMatch(k -> k == hour)) {
continue;
}
JSONObject jsonObject = JSONObject.parseObject(data);
for (Map.Entry<String, Object> entry : jsonObject.entrySet()) {
String k = entry.getKey();
Object v = entry.getValue();
system = k;
boolean lock = RedisLock.tryLock("LOG_CENTER_PHONE_WARN_" + k, phoneWarnLockHour * 60 * 60, 0, TimeUnit.SECONDS);
if (!lock) {
log.info("{}系统在{}小时内重复触发电话告警,程序已拦截", k, phoneWarnLockHour);
continue;
}
JSONArray telWarnTitles = JSONObject.parseArray(v.toString());
String titleStr = telWarnTitles.stream().map(m -> m.toString()).collect(Collectors.joining("。"));
int size = telWarnTitles.size();
String content = String.format("%s共出现%s条超过阈值错误信息%s", k, size, titleStr);
MOBILE_PHONE.stream().forEach(m -> ntfServiceClient.callAlarmPhone(m, k, content));
}
} catch (Exception e) {
log.error("{}超过电话预警获取延迟队列出现异常e:{}", system, e);
}
}
}).start();
}
@SneakyThrows
@Override
public TaskResponse execute(TaskRequest request) {
log.info("开始执行错误日志top10巡检");
//1.模拟登录 jsoup
jsoupLogin();
for (String systemName : SEARCH_SYSTEM) {
try {
Thread.sleep(1000);
//2.获取key
String id = getKey(systemName);
Thread.sleep(8000);
//3.获取查询结果
List<Map<String, Object>> logCenterList = retryUtil.doRetry(4, () -> getLogContents(systemName, id), systemName + "获取日志数据");
if (CollectionUtils.isEmpty(logCenterList)) {
log.error("{}发送内容为空,不发送机器人微信消息", systemName);
continue;
}
//4.处理信息
handleMessage(systemName, logCenterList);
} catch (Exception e) {
log.error("{}发送错误日志告警出现异常e:{}", systemName, e);
}
}
telWarnIndex = 0;
return new TaskResponse(TaskResponse.SUCCESS, "logCenter任务完成");
}
private List<Map<String, Object>> getLogContents(String systemName, String id) throws Exception {
List<Map<String, Object>> contentList = getContentList(systemName, id);
if (CollectionUtils.isEmpty(contentList)) {
Thread.sleep(8000);
throw new Exception(systemName + "未查询到数据,需要重试!");
}
return contentList;
}
public String getKey(String systemName) {
Date endDate = new Date();
Date startDate = DateUtil.offsetHour(endDate, -PERIOD_HOUR);
//请求参数
JSONObject paramJson = JSON.parseObject(String.format(QUERY_TEMPLATE, systemName, systemName));
paramJson.put("startTime", startDate.getTime());
paramJson.put("endTime", endDate.getTime());
HttpResponse execute = HttpRequest.post(LOG_CENTER_BASE_URL + "jobs")
//设置请求头(可任意加)
.header("X-Csrf-Token", X_Csrf_Token)
.header("Cookie", COOKIE)
.header("Content-Type", "application/json")
.header("Connection", "keep-alive")
//请求参数
.body(paramJson.toJSONString())
.timeout(40000)
.execute();
String body1 = execute.body();
String id = JSON.parseObject(body1).getString("id");
return id;
}
public List<Map<String, Object>> getContentList(String systemName, String id) {
String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";
HttpResponse execute = HttpRequest.get(url)
//设置请求头(可任意加)
.header("X-Csrf-Token", X_Csrf_Token)
.header("Cookie", COOKIE)
//.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
.timeout(40000)
.execute();
JSONArray rows = JSON.parseObject(execute.body()).getJSONArray("rows");
System.err.println(rows.toString());
List<Map<String, Object>> list = Lists.newArrayList();
for (Object row : rows) {
try {
JSONArray jsonArray = JSON.parseArray(row.toString());
String title = jsonArray.get(0).toString();
Integer count = Integer.valueOf(jsonArray.get(1).toString());
String content = jsonArray.get(2).toString();
if (count >= maxCount) {
continue;
}
if (filterContent.stream().anyMatch(k -> content.contains(k))) {
continue;
}
int subContentLen = content.length() <= SUB_CONTENT_LENGTH ? content.length() : SUB_CONTENT_LENGTH;
String subContent = content.substring(0, subContentLen);
int subTitleLen = title.length() <= SUB_TITLE_LENGTH ? title.length() : SUB_TITLE_LENGTH;
String subTitle = title.substring(0, subTitleLen);
Map<String, Object> map = new HashMap<>();
map.put("title", subTitle);
map.put("count", count);
map.put("content", StrUtil.removeSuffix(subContent, "{"));
list.add(map);
} catch (Exception e) {
log.error("{}获取日志统计数据出现异常e:{}", systemName, e);
}
}
if (CollectionUtils.isEmpty(list)) {
log.info("{}未获取到数据,请求链接url:{},token:{},cookie:{}", systemName, url, X_Csrf_Token, COOKIE);
}
return list;
}
public void handleMessage(String systemName, List<Map<String, Object>> contentList) {
int topNum = 1;
int warnTitleNum = 1;
List<String> wxContents = Lists.newArrayList();
List<String> telWarnTitles = Lists.newArrayList();
for (Map<String, Object> k : contentList) {
String title = (String) k.get("title");
Integer count = (Integer) k.get("count");
Boolean warnFlag = Optional.ofNullable(count).map(m -> m > warnCount).orElse(false);
String content = (String) k.get("content");
String wxContent;
if (warnFlag) {
wxContent = String.format("> <font color=warning>top-%d:出现次数:%d</font>\n> <font color=warning>错误详情描述: %s</font>", topNum++, count, content.trim());
String warnTitle = String.format("第%s条%s", warnTitleNum++, title);
telWarnTitles.add(warnTitle);
} else {
wxContent = String.format("> top-%d:出现次数:%d\n> 错误详情描述: %s", topNum++, count, content.trim());
}
wxContents.add(wxContent);
}
sendWxMsg(systemName, wxContents);
if (CollectionUtils.isNotEmpty(telWarnTitles) && PERIOD_HOUR <= 24) {
JSONObject jsonObject = new JSONObject();
jsonObject.put(systemName, telWarnTitles);
try {//入延迟队列,为打电话做准备
delayQueue.put(new DelayedElement(jsonObject.toJSONString(), 120000 * telWarnIndex));//n*2分钟
telWarnIndex++;
} catch (Exception e) {
log.error("{}超过电话预警加入延迟队列出现异常e:{}", systemName, e);
}
}
}
private void sendWxMsg(String systemName, List<String> wxContents) {
String periodDesc = getPeriodDesc();
String StringContent = wxContents.stream().filter(Objects::nonNull).collect(Collectors.joining("\n\n"));
String content = String.format("%s近%s内错误日志Top10\n%s", systemName, periodDesc, StringContent);
String sendContent = String.format(TEMPLATE, content);//
WX_ROBOT_URL.forEach(wx -> {
String post2 = HttpUtil.post(wx, sendContent);
String errcode = JSON.parseObject(post2).getString("errcode");
if (!"0".equals(errcode)) {
log.error("{}发送微信不成功,返回数据:{}", systemName, post2);
}
});
}
private String getPeriodDesc() {
if (PERIOD_HOUR <= 24) {
return PERIOD_HOUR + "小时";
}
BigDecimal dayBigDecimal = BigDecimal.valueOf(PERIOD_HOUR).divide(BigDecimal.valueOf(24), 2, BigDecimal.ROUND_HALF_UP);
String s = StrUtil.removeSuffix(StrUtil.removeSuffix(String.valueOf(dayBigDecimal), "00"), "0");
String[] split = s.split("\\.");
if (split.length == 1) {
return split[0] + "天";
}
return s + "天";
}
/**
* Jsoup 模拟登录 访问个人中心
* 在登录时先输入一个错误的账号密码,查看到登录所需要的参数
* 先构造登录请求参数,成功后获取到cookies
* 设置request cookies,再次请求
*
* @throws IOException
*/
public void jsoupLogin() throws IOException {
//Jsoup加这个,避免请求https报证书问题
trustEveryone();
// 构造登陆参数
Map<String, String> data = new HashMap<>();
data.put("username", USER_NAME);
data.put("password", PASSWORD);
Connection.Response response = Jsoup.connect(LOG_CENTER_BASE_URL + "account/ldap/login")
.ignoreContentType(true) // 忽略类型验证
.ignoreHttpErrors(true)
.followRedirects(false) // 禁止重定向
.postDataCharset("utf-8")
.header("Upgrade-Insecure-Requests", "1")
.header("Accept", "application/json")
.header("Content-Type", "application/json")
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36")
.requestBody(JSON.toJSONString(data))
.method(Connection.Method.POST)
.execute();
response.charset("UTF-8");
// login 中已经获取到登录成功之后的cookies
// 构造访问个人中心的请求
Map<String, String> cookies = response.cookies();
cookies.forEach((k, v) -> COOKIE = k + "=" + v);
String body = response.body();
X_Csrf_Token = JSON.parseObject(body).getString("X-Csrf-Token");
System.err.println("COOKIE:" + COOKIE);
System.err.println("X_Csrf_Token:" + X_Csrf_Token);
}
public void jsoupHandle(String id) throws IOException {
String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";
Document document = Jsoup.connect(url)
.header("X-Csrf-Token", X_Csrf_Token)
.header("Cookie", COOKIE)
.ignoreContentType(true) // 忽略类型验证
.ignoreHttpErrors(true)
.method(Connection.Method.GET)
.get();
System.err.println(url);
String s = JSON.toJSONString(document);
System.err.println(s);
}
public static void trustEveryone() {
try {
HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
public boolean verify(String hostname, SSLSession session) {
return true;
}
});
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, new X509TrustManager[]{new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}}, new SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
} catch (Exception e) {
// e.printStackTrace();
}
}
}
package com.smy.cbs.entity;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
public class DelayedElement implements Delayed {
private String data;
private long delayTime;
public DelayedElement(String data, long delayTime) {
this.data = data;
this.delayTime = System.currentTimeMillis() + delayTime;
}
@Override
public long getDelay(TimeUnit unit) {
long diff = delayTime - System.currentTimeMillis();
return unit.convert(diff, TimeUnit.MILLISECONDS);
}
@Override
public int compareTo(Delayed o) {
long diff = this.getDelay(TimeUnit.MILLISECONDS) - o.getDelay(TimeUnit.MILLISECONDS);
return Long.compare(diff, 0);
}
public String getData() {
return data;
}
}
三、实现效果
企业微信机器狗开发者文档:群机器人配置说明 - 接口文档 - 企业微信开发者中心