java阿里云sls基于LoghubAppender自定义日志上传

1、背景：阿里sls日志提供快捷日志平台，平替elk公司使用这个日志服务，需要对接写入日志

目前日志集成有3种

        1）基于封装manager手动写日志手动send

              弊端：本地日志和阿里云日志共用日志代码很臃肿

        2）基于云服务器日志文件写入日志平台

                弊端：日志分割太碎，一次请求分割很多条日志

        3）代码LoghubAppender写入日志平台

                弊端：日志分割太碎，一次请求分割很多条日志

2、期望实现效果：日志按请求维度发送

3、实现思路：

      日志发送在LoghubAppender里面appendEvent(E eventObject)方法里面实现了日志内组装和发送，想法是拆分日志组装和发送，先组装不发送，在请求完成后再发送

3、实现方式：

自定义LoghubAppender

自定义log写入类:

完整appendEvent方法:

private void appendEvent(E eventObject) {
        if (eventObject instanceof LoggingEvent) {
            LoggingEvent event = (LoggingEvent) eventObject;
            List<LogItem> logItems = new ArrayList<>();
            LogItem item = new LogItem();
            logItems.add(item);
            item.SetTime((int) (event.getTimeStamp() / 1000L));
            if (this.formatter != null) {
                DateTime dateTime = new DateTime(event.getTimeStamp());
                item.PushBack("time", dateTime.toString(this.formatter));
            } else {
                Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
                item.PushBack("time", this.formatter1.format(instant));
            }

            item.PushBack("level", event.getLevel().toString());
            item.PushBack("thread", event.getThreadName());
            StackTraceElement[] caller = event.getCallerData();
            if (caller != null && caller.length > 0) {
                item.PushBack("location", caller[0].toString());
            }

            String message = event.getFormattedMessage();
            item.PushBack("message", message);
            IThrowableProxy iThrowableProxy = event.getThrowableProxy();
            if (iThrowableProxy != null) {
                String throwable = this.getExceptionInfo(iThrowableProxy);
                throwable = throwable + this.fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
                item.PushBack("throwable", throwable);
            }

            if (this.encoder != null) {
                item.PushBack("log", new String(this.encoder.encode(eventObject)));
            }

            Optional.ofNullable(this.mdcFields).ifPresent(f -> event.getMDCPropertyMap().entrySet().stream()
                    .filter(v -> Arrays.stream(f.split(",")).anyMatch(i -> i.equals(v.getKey())))
                    .forEach(map -> item.PushBack(map.getKey(), map.getValue())));
            AliyunLogManager aliyunLogManager = null;
            try {
                aliyunLogManager = SpringContext.getBean(AliyunLogManager.class);
            } catch (Exception e) {
                log.info("AliyunLogManager bean has not initialized");
            }

            if (aliyunLogManager != null && MDC.get(CommonConst.LOG_TRACE_ID) != null) {
                aliyunLogManager.writeLog("log", logItemToString(logItems));
            } else {
                try {
                    this.producer.send(this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems
                            , new LoghubAppenderCallback(this, this.projectConfig.getProject(), this.logStore, this.topic, this.source, logItems));
                } catch (Exception var9) {
                    this.addError("Failed to send log, project=");
                }
            }

        }
    }

阿里云日志工具类: 

主要思路：写日志时候放至threadlocal中

import com.aliyun.openservices.aliyun.log.producer.LogProducer;
import com.aliyun.openservices.aliyun.log.producer.Producer;
import com.aliyun.openservices.aliyun.log.producer.ProducerConfig;
import com.aliyun.openservices.aliyun.log.producer.ProjectConfig;
import com.aliyun.openservices.aliyun.log.producer.errors.ProducerException;
import com.aliyun.openservices.log.common.LogItem;
import com.google.common.collect.Lists;
import com.ty.mid.common.base.thread.TyThreadPool;
import com.ty.mid.common.base.utils.LocalDateTimeUtil;
import com.ty.mid.common.log.config.AliYunAutoConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

/**
 * @author hll
 * @date 2022/9/20
 * 阿里云日志工具类
 */
@Component
@Slf4j
public class AliyunLogManager implements InitializingBean, DisposableBean {

    @Autowired
    private AliYunAutoConfig aliYunAutoConfig;

    private Producer producer;

    public static final ThreadLocal<List<Map<String, String>>> LOCAL_LOG = new ThreadLocal<>();
    public static final ThreadLocal<List<Map<String, String>>> LOCAL_TAG = new ThreadLocal<>();

    @Override
    public void afterPropertiesSet() {
        ProducerConfig producerConfig = new ProducerConfig();
        producerConfig.setBatchSizeThresholdInBytes(3145728);
        producerConfig.setBatchCountThreshold(40960);
        producerConfig.setIoThreadCount(this.aliYunAutoConfig.getThreadCount());
        producerConfig.setTotalSizeInBytes(this.aliYunAutoConfig.getTotalSize());
        this.producer = new LogProducer(producerConfig);
        this.producer.putProjectConfig(new ProjectConfig(this.aliYunAutoConfig.getProject(), this.aliYunAutoConfig.getEndpoint()
                , this.aliYunAutoConfig.getAccessKey(), this.aliYunAutoConfig.getAccessSecret()));
    }

    @Override
    public void destroy() throws Exception {
        if (this.producer != null) {
            try {
                this.producer.close();
            } catch (ProducerException var2) {
                log.error("Failed to close producer, e=", var2);
            }
        }

    }

    public void writeLog(String title, String value) {
        List<Map<String, String>> logList = LOCAL_LOG.get();
        if (logList == null) {
            logList = new ArrayList<>();
        }

        Map<String, String> logMap = new HashMap<>(1);
        logMap.put(String.format("%s_%s", LocalDateTimeUtil.getSystemDateStr(), title), value);
        (logList).add(logMap);
        LOCAL_LOG.set(logList);
    }

    public void setTag(String tagName, String tagValue) {
        List<Map<String, String>> logList = LOCAL_TAG.get();
        if (logList == null) {
            logList = new ArrayList<>();
        }

        Map<String, String> logMap = new HashMap<>(1);
        logMap.put(String.format("__tag__:__%s__", tagName), tagValue);
        (logList).add(logMap);
        LOCAL_TAG.set(logList);
    }

    public void send() {
        List<Map<String, String>> logList = LOCAL_LOG.get();
        List<Map<String, String>> tagList = LOCAL_TAG.get();
        //防止日志数量过大 阿里日志拒绝写入/读取
        List<List<Map<String, String>>> partition = Lists.partition(CollectionUtils.isEmpty(logList) ? new ArrayList<>() : logList, 500);
        for (List<Map<String, String>> maps : partition) {
            handleSend(maps, tagList);
        }
        LOCAL_LOG.remove();
        LOCAL_TAG.remove();
    }

    private void handleSend(List<Map<String, String>> logList, List<Map<String, String>> tagList) {
        if (CollectionUtils.isNotEmpty(logList)) {
            final List<Map<String, String>> logListCopy = new ArrayList<>();
            int i = 0;
            Map<String, String> map;

            for (Map<String, String> log : logList) {
                map = new HashMap<>(log.size());

                for (String key : log.keySet()) {
                    StringBuilder var10001 = (new StringBuilder()).append(key).append("_");
                    ++i;
                    if (i < 10) {
                        map.put(var10001.append("0").append(i).toString(), log.get(key));
                    } else {
                        map.put(var10001.append(i).toString(), log.get(key));
                    }

                }

                logListCopy.add(map);
            }

            //tag补充
            if (CollectionUtils.isNotEmpty(tagList)) {
                for (Map<String, String> log : tagList) {
                    map = new HashMap<>(log.size());

                    for (String key : log.keySet()) {
                        StringBuilder var10001 = (new StringBuilder()).append(key).append("_");
                        ++i;
                        if (i < 10) {
                            map.put(var10001.append("0").append(i).toString(), log.get(key));
                        } else {
                            map.put(var10001.append(i).toString(), log.get(key));
                        }
                    }

                    logListCopy.add(map);
                }
            }


            ThreadPool.execute(() -> {
                try {
                    this.producer.send(this.aliYunAutoConfig.getProject(), this.aliYunAutoConfig.getLogStore()
                            , this.aliYunAutoConfig.getTopic(), this.aliYunAutoConfig.getSource(), this.generateLogItem(logListCopy), result -> {
                                if (!result.isSuccessful()) {
                                    log.error("send log failed");
                                }
                            });
                } catch (Exception var2) {
                    log.error(var2.getMessage(), var2);
                }
            });
        }
    }

    private LogItem generateLogItem(List<Map<String, String>> logList) {
        LogItem logItem = new LogItem();
        Iterator<Map<String, String>> var3 = logList.iterator();
        while (var3.hasNext()) {
            Map logMap = var3.next();
            for (Object o : logMap.entrySet()) {
                Map.Entry<String, String> entry = (Map.Entry) o;
                logItem.PushBack(entry.getKey(), entry.getValue());
            }
        }

        return logItem;
    }
}

基于请求维度发送日志（logback日志为例）：

/**
 * @author hll
 * @date 2022/9/27
 * logback 阿里云日志对接处理
 */
@Component
@Order(-1)
public class LogbackRequestLoggingFilter extends AbstractRequestLoggingFilter {

    @Autowired
    private AliyunLogManager aliyunLogManager;

    @Override
    protected void beforeRequest(HttpServletRequest httpServletRequest, String s) {
        //http请求标记 便于直接按请求打印日志
        aliyunLogManager.setTag("traceId", MDC.get(CommonConst.LOG_TRACE_ID));
        aliyunLogManager.setTag("url", httpServletRequest.getRequestURI());
    }

    @Override
    protected void afterRequest(HttpServletRequest httpServletRequest, String s) {
        //set send
        aliyunLogManager.send();
    }
}

写日志：log.info即可 

最终效果：

另外：也实现了基于自定义注解切面的方式发送请求维度日志用于定时任务使用场景

待完善：父子线程/异步线程日志合并处理

如有其他思路，欢迎多交流！