jsoup登录日志平台后调企业微信机器人自动发送错误日志告警

一、需求:错误日志Top10告警发送

二、需求分解

  1. jsoup实现登录,获取到cookie和token等用户鉴权信息
  2. 获取接口相应的key值
  3. 调用日志平台错误日志Top榜接口,查询到结果集
  4. 处理消息(调用企业微信机器人发送消息接口,电话预警功能)
  5. 加上定时任务,可以实现定时发送错误日志告警的功能

定时任务高度设置1小时(根据实际需要)执行一次,可以实现自动巡检错误日志的功能。以便及时发现生产问题

jsoup是java的爬虫框架,可以爬取网页数据,这里没有重点使用,只是做了个登录功能。后续可以专门它写一份爬虫的程序。。

  <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.12.1</version>
        </dependency>
package com.smy.cbs.task;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.smy.cbs.client.NtfServiceClient;
import com.smy.cbs.entity.DelayedElement;
import com.smy.cbs.util.RetryUtil;
import com.smy.framework.redis.RedisLock;
import com.smy.smyx.scheduler.dto.TaskRequest;
import com.smy.smyx.scheduler.dto.TaskResponse;
import com.smy.smyx.scheduler.service.TaskService;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.net.ssl.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
 * @author youlu
 * @ClassName logCenterTask
 * @Date 2023/11/22 17:14
 * @Version V1.0
 **/
@Slf4j
@Service("logCenterTask")
public class LogCenterTask implements TaskService {
    @Resource
    private RetryUtil retryUtil;
    @Resource
    private NtfServiceClient ntfServiceClient;

    @Value("${log.usename:aaaa}")
    public String USER_NAME;
    @Value("${log.password:bbbb}")
    public String PASSWORD;
    @Value("${log.content.warn.max:24000000}")
    public int maxCount;//达到阈值则告警
    @Value("${log.warn.count:100}")
    public int warnCount;//字体标黄,打电话告警阈值
    @Value("#{'${log.content.filter.content:输入要过滤的内容有英文逗号隔开}'.split(',')}")
    public List<String> filterContent = Lists.newArrayList();//过滤内容
    @Value("${log.content.length:240}")
    public int SUB_CONTENT_LENGTH = 300;//截取报错内容字符串长度
    @Value("${log.title.length:20}")
    public int SUB_TITLE_LENGTH = 40;//截取报错类型字符串长度
    @Value("${log.period.hour:24}")
    public int PERIOD_HOUR;//24小时内的错误日志
    @Value("#{'${log.search.system:cbs_core,rls_core,uts_core,adv_core}'.split(',')}")
    public List<String> SEARCH_SYSTEM;//查询的系统
    @Value("#{'${log.wx.url:https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=b54887d8-xxxx,https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=87140cb7-x123}'.split(',')}")
    public List<String> WX_ROBOT_URL;
    @Value("#{'${log.warn.phone:}'.split(',')}")
    public List<String> MOBILE_PHONE;//预警手机
    @Value("#{'${log.filter.phone.time:0,1,2,3,4,5,6,7,13,22,23}'.split(',')}")
    public List<Integer> filterPhoneTime;//电话预警时间过滤
    @Value("${log.phone.warn.lock.hour:6}")
    public int phoneWarnLockHour;//小时
    @Value("${log.url:https://log_center.xxxx.com/api/}")
    public  String LOG_CENTER_BASE_URL;

    public static String X_Csrf_Token = "59e82a89-d671-462e-87be-ee6cd30cd7f8";
    public static String COOKIE = "JSESSIONID=MWY1NjUxMDItYzU3Ny00Y2Y1LTk2OTYtZjUxZjQ5ZGIzZWRiLGwwMTU2MixyNmdNVkxsSHJWNW9kaDFoMTMxVGRNQ0oyc3c9";
    public static String id = "Zf_0-osBnsFDyEFyLnk0";

    public static final String TEMPLATE = "{\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"%s\"}}";
    public static final String QUERY_TEMPLATE = "{\n" +
            "  \"app\": \"system\",\n" +
            "  \"source\": \"other\",\n" +
            "  \"query\": \"repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n| where level=\\\"ERROR\\\"\\n| eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n| stats count() as num by err_type\\n| sort 10 by num\\n| join type=inner err_type [\\n  repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n  | where level=\\\"ERROR\\\"\\n  | eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n  | fields + err_type, _raw\\n  | dedup err_type\\n]\\n| rename _raw as 原始日志, num as 统计, err_type as 错误类型\",\n" +
            "  \"mode\": \"smart\",\n" +
            "  \"preview\": false,\n" +
            "  \"collectSize\": -1,\n" +
            "  \"timeout\": 1000,\n" +
            "  \"sorts\": []\n" +
            "}";
    public static final BlockingQueue<DelayedElement> delayQueue = new DelayQueue<>();
    public static int telWarnIndex = 0;


    @PostConstruct
    public void init() {
        new Thread(() -> {
            while (true) {
                String  system = null;
                try {
                    DelayedElement element = delayQueue.take();
                    String data = element.getData();
                    if (StringUtils.isBlank(data)) {
                        continue;
                    }
                    Date date = new Date();
                    int hour = DateUtil.hour(date, true);
                    if (filterPhoneTime.stream().anyMatch(k -> k == hour)) {
                        continue;
                    }
                    JSONObject jsonObject = JSONObject.parseObject(data);
                    for (Map.Entry<String, Object> entry : jsonObject.entrySet()) {
                        String k = entry.getKey();
                        Object v = entry.getValue();
                        system = k;
                        boolean lock = RedisLock.tryLock("LOG_CENTER_PHONE_WARN_" + k, phoneWarnLockHour * 60 * 60, 0, TimeUnit.SECONDS);
                        if (!lock) {
                            log.info("{}系统在{}小时内重复触发电话告警,程序已拦截", k, phoneWarnLockHour);
                            continue;
                        }
                        JSONArray telWarnTitles = JSONObject.parseArray(v.toString());
                        String titleStr = telWarnTitles.stream().map(m -> m.toString()).collect(Collectors.joining("。"));
                        int size = telWarnTitles.size();
                        String content = String.format("%s共出现%s条超过阈值错误信息%s", k, size, titleStr);
                        MOBILE_PHONE.stream().forEach(m -> ntfServiceClient.callAlarmPhone(m, k, content));
                    }
                } catch (Exception e) {
                    log.error("{}超过电话预警获取延迟队列出现异常e:{}", system, e);
                }
            }
        }).start();
    }

    @SneakyThrows
    @Override
    public TaskResponse execute(TaskRequest request) {
        log.info("开始执行错误日志top10巡检");
        //1.模拟登录 jsoup
        jsoupLogin();

        for (String systemName : SEARCH_SYSTEM) {
            try {
                Thread.sleep(1000);
                //2.获取key
                String id = getKey(systemName);
                Thread.sleep(8000);
                //3.获取查询结果
                List<Map<String, Object>> logCenterList = retryUtil.doRetry(4, () -> getLogContents(systemName, id), systemName + "获取日志数据");
                if (CollectionUtils.isEmpty(logCenterList)) {
                    log.error("{}发送内容为空,不发送机器人微信消息", systemName);
                    continue;
                }
                //4.处理信息
                handleMessage(systemName, logCenterList);
            } catch (Exception e) {
                log.error("{}发送错误日志告警出现异常e:{}", systemName, e);
            }
        }
        telWarnIndex = 0;
        return new TaskResponse(TaskResponse.SUCCESS, "logCenter任务完成");
    }

    private List<Map<String, Object>> getLogContents(String systemName, String id) throws Exception {
        List<Map<String, Object>> contentList = getContentList(systemName, id);
        if (CollectionUtils.isEmpty(contentList)) {
            Thread.sleep(8000);
            throw new Exception(systemName + "未查询到数据,需要重试!");
        }
        return contentList;
    }

    public String getKey(String systemName) {
        Date endDate = new Date();
        Date startDate = DateUtil.offsetHour(endDate, -PERIOD_HOUR);
        //请求参数
        JSONObject paramJson = JSON.parseObject(String.format(QUERY_TEMPLATE, systemName, systemName));
        paramJson.put("startTime", startDate.getTime());
        paramJson.put("endTime", endDate.getTime());
        HttpResponse execute = HttpRequest.post(LOG_CENTER_BASE_URL + "jobs")
                //设置请求头(可任意加)
                .header("X-Csrf-Token", X_Csrf_Token)
                .header("Cookie", COOKIE)
                .header("Content-Type", "application/json")
                .header("Connection", "keep-alive")
                //请求参数
                .body(paramJson.toJSONString())
                .timeout(40000)
                .execute();
        String body1 = execute.body();
        String id = JSON.parseObject(body1).getString("id");
        return id;
    }

    public List<Map<String, Object>> getContentList(String systemName, String id) {
        String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";
        HttpResponse execute = HttpRequest.get(url)
                //设置请求头(可任意加)
                .header("X-Csrf-Token", X_Csrf_Token)
                .header("Cookie", COOKIE)
                //.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
                .timeout(40000)
                .execute();

        JSONArray rows = JSON.parseObject(execute.body()).getJSONArray("rows");
        System.err.println(rows.toString());
        List<Map<String, Object>> list = Lists.newArrayList();
        for (Object row : rows) {
            try {
                JSONArray jsonArray = JSON.parseArray(row.toString());
                String title = jsonArray.get(0).toString();
                Integer count = Integer.valueOf(jsonArray.get(1).toString());
                String content = jsonArray.get(2).toString();
                if (count >= maxCount) {
                    continue;
                }
                if (filterContent.stream().anyMatch(k -> content.contains(k))) {
                    continue;
                }
                int subContentLen = content.length() <= SUB_CONTENT_LENGTH ? content.length() : SUB_CONTENT_LENGTH;
                String subContent = content.substring(0, subContentLen);
                int subTitleLen = title.length() <= SUB_TITLE_LENGTH ? title.length() : SUB_TITLE_LENGTH;
                String subTitle = title.substring(0, subTitleLen);

                Map<String, Object> map = new HashMap<>();
                map.put("title", subTitle);
                map.put("count", count);
                map.put("content", StrUtil.removeSuffix(subContent, "{"));
                list.add(map);
            } catch (Exception e) {
                log.error("{}获取日志统计数据出现异常e:{}", systemName, e);
            }
        }
        if (CollectionUtils.isEmpty(list)) {
            log.info("{}未获取到数据,请求链接url:{},token:{},cookie:{}", systemName, url, X_Csrf_Token, COOKIE);
        }
        return list;
    }

    public void handleMessage(String systemName, List<Map<String, Object>> contentList) {
        int topNum = 1;
        int warnTitleNum = 1;
        List<String> wxContents = Lists.newArrayList();
        List<String> telWarnTitles = Lists.newArrayList();
        for (Map<String, Object> k : contentList) {
            String title = (String) k.get("title");
            Integer count = (Integer) k.get("count");
            Boolean warnFlag = Optional.ofNullable(count).map(m -> m > warnCount).orElse(false);
            String content = (String) k.get("content");
            String wxContent;
            if (warnFlag) {
                wxContent = String.format("> <font color=warning>top-%d:出现次数:%d</font>\n> <font color=warning>错误详情描述: %s</font>", topNum++, count, content.trim());
                String warnTitle = String.format("第%s条%s", warnTitleNum++, title);
                telWarnTitles.add(warnTitle);
            } else {
                wxContent = String.format("> top-%d:出现次数:%d\n> 错误详情描述: %s", topNum++, count, content.trim());
            }
            wxContents.add(wxContent);
        }
        sendWxMsg(systemName, wxContents);
        if (CollectionUtils.isNotEmpty(telWarnTitles) && PERIOD_HOUR <= 24) {
            JSONObject jsonObject = new JSONObject();
            jsonObject.put(systemName, telWarnTitles);
            try {//入延迟队列,为打电话做准备
                delayQueue.put(new DelayedElement(jsonObject.toJSONString(), 120000 * telWarnIndex));//n*2分钟
                telWarnIndex++;
            } catch (Exception e) {
                log.error("{}超过电话预警加入延迟队列出现异常e:{}", systemName, e);
            }
        }
    }

    private void sendWxMsg(String systemName, List<String> wxContents) {
        String periodDesc = getPeriodDesc();
        String StringContent = wxContents.stream().filter(Objects::nonNull).collect(Collectors.joining("\n\n"));
        String content = String.format("%s近%s内错误日志Top10\n%s", systemName, periodDesc, StringContent);
        String sendContent = String.format(TEMPLATE, content);//
        WX_ROBOT_URL.forEach(wx -> {
            String post2 = HttpUtil.post(wx, sendContent);
            String errcode = JSON.parseObject(post2).getString("errcode");
            if (!"0".equals(errcode)) {
                log.error("{}发送微信不成功,返回数据:{}", systemName, post2);
            }
        });
    }


    private String getPeriodDesc() {
        if (PERIOD_HOUR <= 24) {
            return PERIOD_HOUR + "小时";
        }
        BigDecimal dayBigDecimal = BigDecimal.valueOf(PERIOD_HOUR).divide(BigDecimal.valueOf(24), 2, BigDecimal.ROUND_HALF_UP);
        String s = StrUtil.removeSuffix(StrUtil.removeSuffix(String.valueOf(dayBigDecimal), "00"), "0");
        String[] split = s.split("\\.");
        if (split.length == 1) {
            return split[0] + "天";
        }
        return s + "天";
    }

    /**
     * Jsoup 模拟登录 访问个人中心
     * 在登录时先输入一个错误的账号密码,查看到登录所需要的参数
     * 先构造登录请求参数,成功后获取到cookies
     * 设置request cookies,再次请求
     *
     * @throws IOException
     */
    public void jsoupLogin() throws IOException {
        //Jsoup加这个,避免请求https报证书问题
        trustEveryone();
        // 构造登陆参数
        Map<String, String> data = new HashMap<>();
        data.put("username", USER_NAME);
        data.put("password", PASSWORD);
        Connection.Response response = Jsoup.connect(LOG_CENTER_BASE_URL + "account/ldap/login")
                .ignoreContentType(true) // 忽略类型验证
                .ignoreHttpErrors(true)
                .followRedirects(false) // 禁止重定向
                .postDataCharset("utf-8")
                .header("Upgrade-Insecure-Requests", "1")
                .header("Accept", "application/json")
                .header("Content-Type", "application/json")
                .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36")
                .requestBody(JSON.toJSONString(data))
                .method(Connection.Method.POST)
                .execute();
        response.charset("UTF-8");
        // login 中已经获取到登录成功之后的cookies
        // 构造访问个人中心的请求
        Map<String, String> cookies = response.cookies();
        cookies.forEach((k, v) -> COOKIE = k + "=" + v);
        String body = response.body();
        X_Csrf_Token = JSON.parseObject(body).getString("X-Csrf-Token");

        System.err.println("COOKIE:" + COOKIE);
        System.err.println("X_Csrf_Token:" + X_Csrf_Token);
    }

    public  void jsoupHandle(String id) throws IOException {
        String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";
        Document document = Jsoup.connect(url)
                .header("X-Csrf-Token", X_Csrf_Token)
                .header("Cookie", COOKIE)
                .ignoreContentType(true) // 忽略类型验证
                .ignoreHttpErrors(true)
                .method(Connection.Method.GET)
                .get();
        System.err.println(url);
        String s = JSON.toJSONString(document);
        System.err.println(s);

    }


    public static void trustEveryone() {
        try {
            HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
                public boolean verify(String hostname, SSLSession session) {
                    return true;
                }
            });

            SSLContext context = SSLContext.getInstance("TLS");
            context.init(null, new X509TrustManager[]{new X509TrustManager() {
                public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[0];
                }
            }}, new SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
        } catch (Exception e) {
            // e.printStackTrace();
        }
    }

}
package com.smy.cbs.entity;

import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;

public class DelayedElement implements Delayed {
    private String data;
    private long delayTime;

    public DelayedElement(String data, long delayTime) {
        this.data = data;
        this.delayTime = System.currentTimeMillis() + delayTime;
    }

    @Override
    public long getDelay(TimeUnit unit) {
        long diff = delayTime - System.currentTimeMillis();
        return unit.convert(diff, TimeUnit.MILLISECONDS);
    }

    @Override
    public int compareTo(Delayed o) {
        long diff = this.getDelay(TimeUnit.MILLISECONDS) - o.getDelay(TimeUnit.MILLISECONDS);
        return Long.compare(diff, 0);
    }

    public String getData() {
        return data;
    }
}

三、实现效果

企业微信机器狗开发者文档:群机器人配置说明 - 接口文档 - 企业微信开发者中心

要使用jsoup登录带验证码的网站,需要进行以下步骤: 1. 首先,使用jsoup获取登录页面的HTML源代码。 2. 解析HTML源代码,找到验证码图片的地址,并将其下载到本地。 3. 使用第三方库,如Tess4J,对验证码图片进行识别,获取验证码的文本信息。 4. 构建登录请求,将用户名、密码、验证码等信息作为参数提交给服务器。 5. 解析服务器返回的响应,判断登录是否成功。 以下是一个简单的示例代码,可以帮助你实现登录带验证码的网站: ``` import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.File; public class LoginDemo { public static void main(String[] args) throws Exception { // 获取登录页面的HTML源代码 Connection.Response response = Jsoup.connect("https://example.com/login") .method(Connection.Method.GET) .execute(); Document doc = response.parse(); // 下载验证码图片到本地 String captchaUrl = doc.select("img#captchaImg").attr("src"); Connection.Response resultImageResponse = Jsoup.connect(captchaUrl) .cookies(response.cookies()) .ignoreContentType(true) .execute(); File captchaImageFile = new File("captcha.png"); FileUtils.copyInputStreamToFile(resultImageResponse.bodyStream(), captchaImageFile); // 识别验证码文本 String captchaText = Tess4JOCR.recognize(captchaImageFile); // 构建登录请求 Connection.Response loginResponse = Jsoup.connect("https://example.com/login") .data("username", "your_username") .data("password", "your_password") .data("captcha", captchaText) .cookies(response.cookies()) .method(Connection.Method.POST) .execute(); // 解析服务器返回的响应 Document loginDoc = loginResponse.parse(); if (loginDoc.title().contains("登录成功")) { System.out.println("登录成功!"); } else { System.out.println("登录失败!"); } } } ``` 请注意,这只是一个示例代码,实际情况可能会更复杂。您需要根据具体的网站和验证码识别库进行适当的修改。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值