K8S同一个deployment多个pod向pvc输出logback日志到不同文件

K8S同一个deployment多个pod向pvc输出logback日志到不同文件

1.使用内置变量HOSTNAME

    <property name="LOG_PATH" value="/data"/>


    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <File>${LOG_PATH}/servicelog-rt-${HOSTNAME}.log</File>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${LOG_PATH}/servicelog-%d{yyyyMMdd}-${HOSTNAME}.log.%i</fileNamePattern>
            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <maxFileSize>500MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
            <maxHistory>100</maxHistory>
        </rollingPolicy>

        <layout class="ch.qos.logback.classic.PatternLayout">
            <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%msg%n</Pattern>
        </layout>
    </appender>

2.使用自定义hostname变量

第一种方式k8s主机名过长，若想只截取后缀部分字符串可以参考以下代码

package com.example;

import ch.qos.logback.core.PropertyDefinerBase;

import java.net.InetAddress;
import java.net.UnknownHostException;

/**
 * <p> 功能描述：自定义主机名变量 </p>
 * 参考：https://logback.qos.ch/manual/configuration.html#definingPropsOnTheFly
 *
 * @author ouruyi
 * @version 1.0
 * @date Created in 2022/9/24 23:08
 */
public class CustomHostNamePropertyDefiner extends PropertyDefinerBase {

    /**
     * 最大长度
     */
    private Integer maxLength = 5;

    /**
     * 默认值
     */
    private String defaultValue = "localhost";

    public Integer getMaxLength() {
        return maxLength;
    }

    public void setMaxLength(Integer maxLength) {
        this.maxLength = maxLength;
    }

    public String getDefaultValue() {
        return defaultValue;
    }

    public void setDefaultValue(String defaultValue) {
        this.defaultValue = defaultValue;
    }

    @Override
    public String getPropertyValue() {
        InetAddress ia;
        try {
            ia = InetAddress.getLocalHost();
            String host = ia.getHostName();
            final int length = host.length();
            if (length > maxLength) {
                return host.substring(length - maxLength, length);
            } else {
                return host;
            }
        } catch (UnknownHostException e) {
            e.printStackTrace();
        }
        return defaultValue;
    }
}

以下截取部分logback.xml配置文件，其中maxLength建议填写为5或15，其中5仅使用k8s pod name最后5位，15使用pod-template-hash + 最后5位

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <define name="custom.hostname" scope="system" class="com.example.CustomHostNamePropertyDefiner">
        <!-- k8s pod name 可写5或15 -->
        <maxLength>5</maxLength>
        <defaultValue>localhost</defaultValue>
    </define>


	<!-- 系统日志输出 -->
	<appender name="file_info" class="ch.qos.logback.core.rolling.RollingFileAppender">
	    <file>${log.path}/sys-info-${custom.hostname}.log</file>
        <!-- 循环政策：基于时间创建日志文件 -->
		<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- 日志文件名格式 -->
			<fileNamePattern>${log.path}/sys-info-%d{yyyy-MM-dd}-${custom.hostname}.log</fileNamePattern>
			<!-- 日志最大的历史 60天 -->
			<maxHistory>60</maxHistory>
		</rollingPolicy>
		<encoder>
			<pattern>${log.pattern}</pattern>
		</encoder>
		<filter class="ch.qos.logback.classic.filter.LevelFilter">
            <!-- 过滤的级别 -->
            <level>INFO</level>
            <!-- 匹配时的操作：接收（记录） -->
            <onMatch>ACCEPT</onMatch>
            <!-- 不匹配时的操作：拒绝（不记录） -->
            <onMismatch>DENY</onMismatch>
        </filter>
	</appender>
</configuration> 	

3.自动清理日志

• META-INF/spring.factories

org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.example.CustomLogbackConfiguration

• CustomLogbackConfiguration.java

package com.example;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * @author ryou
 * @since 2022/10/14 13:58
 */
@Configuration
public class CustomLogbackConfiguration {

    @Bean
    public CustomHistoryLogCleaner customHistoryLogCleaner() {
        return new CustomHistoryLogCleaner();
    }

}

• CustomHistoryLogCleaner.java

package com.example;

import ch.qos.logback.classic.AsyncAppender;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.rolling.RollingFileAppender;
import ch.qos.logback.core.rolling.RollingPolicy;
import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
import ch.qos.logback.core.util.OptionHelper;
import org.slf4j.ILoggerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.StopWatch;

import java.io.File;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Iterator;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author ryou
 * @since 2022/10/13 18:54
 */
public class CustomHistoryLogCleaner implements InitializingBean {

    private static final String CUSTOM_HOSTNAME = "custom.hostname";
    public static final Pattern PATTERN = Pattern.compile("%d\\{.+}");
    public static final Logger log = LoggerFactory.getLogger(CustomHistoryLogCleaner.class);

    @Override
    public void afterPropertiesSet() {
        final ExecutorService executorService = Executors.newFixedThreadPool(1);
        executorService.execute(this::handle);
        executorService.shutdown();
    }

    /**
     * 遍历ROOT关联的所有RollingFileAppender
     */
    public void handle() {
        final StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory();
        if (loggerFactory instanceof LoggerContext) {
            final Logger root = loggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
            if (root instanceof ch.qos.logback.classic.Logger) {
                final ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) root;
                final Iterator<Appender<ILoggingEvent>> it = logger.iteratorForAppenders();
                while (it.hasNext()) {
                    final Appender<ILoggingEvent> appender = it.next();
                    if (appender instanceof RollingFileAppender) {
                        final RollingFileAppender<?> fileAppender = (RollingFileAppender<?>) appender;
                        handleFileAppender(fileAppender);
                    } else if (appender instanceof AsyncAppender) {
                        final AsyncAppender asyncAppender = (AsyncAppender) appender;
                        final Iterator<Appender<ILoggingEvent>> attachIt = asyncAppender.iteratorForAppenders();
                        while (attachIt.hasNext()) {
                            final Appender<ILoggingEvent> attachAppender = attachIt.next();
                            if (attachAppender instanceof RollingFileAppender) {
                                final RollingFileAppender<?> attachFileAppender = (RollingFileAppender<?>) attachAppender;
                                handleFileAppender(attachFileAppender);
                            }
                        }
                    }
                }
            }
        }
        stopWatch.stop();
        log.info("历史日志清理耗时{}毫秒", stopWatch.getTotalTimeMillis());
    }

    /**
     * 获取RollingFileAppender滚动策略
     */
    public void handleFileAppender(RollingFileAppender<?> appender) {
        int orphanLogSavePeriod = -1;
        // 清理历史日志
        final RollingPolicy rollingPolicy = appender.getRollingPolicy();
        if (rollingPolicy instanceof TimeBasedRollingPolicy) {
            final TimeBasedRollingPolicy<?> timeBasedRollingPolicy = (TimeBasedRollingPolicy<?>) rollingPolicy;
            final String fileNamePattern = timeBasedRollingPolicy.getFileNamePattern();
            final int maxHistory = timeBasedRollingPolicy.getMaxHistory();
            orphanLogSavePeriod = maxHistory;
            // 启动时清理日志
            if (timeBasedRollingPolicy.isCleanHistoryOnStart()) {
                cleanHistoryLogExpired(fileNamePattern, maxHistory);
            }
        }

        // 清理孤儿日志
        if (orphanLogSavePeriod > 0) {
            final String fileNameInUse = appender.getFile();
            cleanOrphanLogExpired(fileNameInUse, orphanLogSavePeriod);
        }
    }

    /**
     * 清理过期日志
     *
     * @param fileNamePattern 文件匹配模式 /data/interview-realtime/application-%d{yyyyMMdd}-86kx2.log.%i
     * @param maxHistory      最大保存历史天数 200
     */
    private void cleanHistoryLogExpired(String fileNamePattern, int maxHistory) {
        log.info("logback日志自动清理:fileNamePattern=>[{}], maxHistory=>[{}]", fileNamePattern, maxHistory);
        final int index = fileNamePattern.lastIndexOf("/");
        String dir = fileNamePattern.substring(0, index);
        String fileName = fileNamePattern.substring(index + 1);
        Matcher matcher = PATTERN.matcher(fileName);
        if (matcher.find()) {
            String pattern = matcher.group(0);
            int start = matcher.start();
            String fileNamePrefix = fileName.substring(0, start);
            String datePattern = pattern.substring(3, pattern.length() - 1);
            final LocalDateTime now = LocalDateTime.now();
            final LocalDateTime expiredDay = now.minusDays(maxHistory);
            final LocalDateTime min = LocalDateTime.of(1970, 1, 1, 12, 0);
            final String maxDay = expiredDay.format(DateTimeFormatter.ofPattern(datePattern));
            final String minDay = min.format(DateTimeFormatter.ofPattern(datePattern));
            final String maxFileName = fileNamePrefix + maxDay;
            final String minFileName = fileNamePrefix + minDay;
            log.info("logback日志自动清理:minFileName=>[{}], maxFileName=>[{}]", minFileName, maxFileName);
            final File file = new File(dir);
            final File[] files = file.listFiles(e -> e.isFile() && e.getName().startsWith(fileNamePrefix));
            for (File f : files) {
                final String name = f.getName();
                log.info("logback日志自动清理:历史日志文件name=>{}", name);
                if (name.compareTo(minFileName) > 0 && name.compareTo(maxFileName) < 0) {
                    log.info("logback日志自动清理:正在删除历史日志文件{}...", name);
                    f.delete();
                }
            }
        }
    }

    /**
     * 清理孤儿日志
     * 注意自定义变量作用域为system
     *
     * @param fileNameInUse 正在写入的日志文件 /data/interview-realtime/interview-realtime-rt-86kx2.log
     * @param maxHistory    最大保存历史天数 200
     * @see ch.qos.logback.core.joran.action.ActionUtil#setProperty
     */
    private void cleanOrphanLogExpired(String fileNameInUse, int maxHistory) {
        log.info("logback日志自动清理:正在写入的日志文件=>[{}], 最大保存周期=>[{}]", fileNameInUse, maxHistory);
        final LocalDateTime now = LocalDateTime.now();
        final LocalDateTime endDateTime = now.minusDays(maxHistory);
        final long maxExpired = endDateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
        final String customHostName = OptionHelper.getSystemProperty(CUSTOM_HOSTNAME);
        if (Objects.isNull(customHostName)) {
            log.info("logback日志自动清理:找不到自定义变量[{}], 已跳过！", CUSTOM_HOSTNAME);
            return;
        }
        final int index = fileNameInUse.lastIndexOf("/");
        String dir = fileNameInUse.substring(0, index);
        String fileName = fileNameInUse.substring(index + 1);
        final int end = fileName.lastIndexOf(customHostName);
        String fileNamePrefix = fileName.substring(0, end);
        final File file = new File(dir);
        final File[] files = file.listFiles(e -> e.isFile() && e.getName().startsWith(fileNamePrefix));
        for (File f : files) {
            final String name = f.getName();
            if (fileName.equals(name)) {
                continue;
            }
            log.info("logback日志自动清理:孤儿日志文件name=>{}", name);
            final long lastModified = f.lastModified();
            if (lastModified < maxExpired) {
                log.info("logback日志自动清理:正在删除孤儿日志文件{}...", name);
                f.delete();
            }
        }

    }

}

参考:
GlusterFS集群文件系统研究
多个spring boot实例输出logback日志到一个文件导致日志混乱问题
多项目写入同一Logback日志文件导致的滚动混乱问题（修改Logback源码）