抓取NG请求响应日志

log_format upstreaminfo '$the_real_ip - [$the_real_ip] - $remote_user [$time_local]  "$http_x_forwarded_for" "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';

import java.io.*;
import java.math.BigDecimal;
import java.util.HashMap;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 抓取NG请求响应日志，包括请求地址，响应最大时间，响应最小时间，请求次数
 * author: tinalucky
 * date: 2021-06-27
 */
public class ReadNgLogFile {

   public static StringBuffer sb = new StringBuffer("");
   public static HashMap map = new HashMap();
   
   public static void main(String[] args) throws IOException {
      final long stime = System.currentTimeMillis();
      final String opType = "POST";

      final ThreadPoolExecutor cacheTheadPool = new ThreadPoolExecutor(5, 10, 3, TimeUnit.SECONDS,
            new ArrayBlockingQueue(3), new ThreadPoolExecutor.DiscardOldestPolicy());
      cacheTheadPool.execute(new Runnable() {
         @Override
         public void run() {
            filterNgMsg("1", opType);
         }
      });
      cacheTheadPool.execute(new Runnable() {
         @Override
         public void run() {
            filterNgMsg("2", opType);
         }
      });
      cacheTheadPool.shutdown();
      while(true){
         if (cacheTheadPool.isTerminated()) {
            break;
         }
      }

      //filterNgMsg("1", opType);
      //filterNgMsg("2", opType);
      writeFile("margeLogFile", opType, sb);
      final long etime = System.currentTimeMillis();
      System.out.println("耗时：" + (etime - stime));
   }

   public static void filterNgMsg(String logName, String optType) {
      final String fname = "D:\\nglog\\" + logName + ".log";
      final File f = new File(fname);
      BufferedReader br = null;
      try {
         final FileReader fr = new FileReader(f);
         br = new BufferedReader(fr);
         String s;
         while ((s = br.readLine()) != null) {
            final String[] arr = s.split(" ");
            //正则匹配在POST 和 HTTP之间的字符串
            final String pattern = "\\S*"+ optType + "(.*?)HTTP\\S*";
            final Matcher matcher = Pattern.compile(pattern).matcher(s);
            //System.out.println(s);
            while(matcher.find()) {
               final String oriUrl = matcher.group(1);
               final String consumeTime = arr[arr.length -2];
               BigDecimal minTime = BigDecimal.valueOf(0);
               // 获取？号前的地址
               BigDecimal maxTime = BigDecimal.valueOf(0);
               int reqCount = 0;
               final String[] urlArr = oriUrl.split("\\?");
               final String url = urlArr[0];
               BigDecimal curTime = BigDecimal.valueOf(0);
               if ("-".equals(consumeTime) || "".equals(consumeTime)) {
                  curTime = BigDecimal.valueOf(0);
               } else {
                  curTime = new BigDecimal(consumeTime);
               }
               if (map.containsKey(url)) {
                  // 修改最小值_最大值_次数
                  final String val = map.get(url);
                  final String[] valArr = val.split("_");
                  BigDecimal _minTime = new BigDecimal(valArr[0]);
                  BigDecimal _maxTime = new BigDecimal(valArr[1]);
                  int _reqCount = 0;
                  if (valArr.length == 3) {
                     _reqCount = new Integer(valArr[2]);
                  } else {
                     _reqCount = 0;
                  }
                  // 比较最大最小值
                  if(curTime.compareTo(_minTime) < 0) {
                     _minTime = curTime;
                  }
                  if(curTime.compareTo(_maxTime) > 0) {
                     _maxTime = curTime;
                  }
                  _reqCount++;
                  map.put(url, _minTime + "_" + _maxTime + "_" + _reqCount);
               } else {
                  final String val = curTime + "_" + curTime + "_" + 1;
                  map.put(url, val);
               }
               //System.out.println(outputResult);
            }
         }
         // 遍历map写入文件
         map.forEach((k, v)->{
            final String[] valArr = v.split("_");
            System.out.println(optType + "\t" + k + "\t" + "\t" + valArr[0]+ '\t' + valArr[1] + "\t" + valArr[2] + "\n");
            sb.append(k + "\t" + "\t" + valArr[0]+ '\t' + valArr[1] + "\t" + valArr[2] + "\n");});
         br.close();
         fr.close();
      } catch(Exception e) {
         e.printStackTrace();
      } finally {
         if (br != null) {
            try {
               br.close();
            } catch (IOException e) {
               e.printStackTrace();
            }
         }
      }
   }

   /**
    * 写文件
    * @param logName 日志名称
    * @param optType 操作类型
    * @param sb 写入的字符串
    * @throws IOException
    */
   private static void writeFile(String logName, String optType, StringBuffer sb) throws IOException {
      final String toWriteFile = "D:\\nglog\\" + logName + "_" + optType + "_result.log";
      final File wf = new File(toWriteFile);
      if(wf.exists()){
         wf.delete();
      }
      final FileWriter fw = new FileWriter(toWriteFile);
      final BufferedWriter bw = new BufferedWriter(fw);
      bw.write(sb.toString());
      bw.close();
      fw.close();
   }
}