logback 日志脱敏 隐藏PII信息

PII信息全称：Personally identifiable information (PII)

PII is any information about an individual maintained by an agency, including

• any information that can be used to distinguish or trace an individual‘s identity, such as name, social security number, date and place of birth, mother‘s maiden name, or biometric records;

• any other information that is linked or linkable to an individual, such as medical, educational, financial, and employment information.

Link:(https://piwik.pro/blog/what-is-pii-personal-data/)

大概翻译一下就是能够代表一个人身份的一些信息，包括但不限于姓名、社会安全号码、出生日期和地点、母亲的婚前姓氏或生物识别记录。

为了提高网络安全集成评估，需要在日志中屏蔽这些信息。

---

以下是具体实现：

利用logback的Converter，自定义日志格式转换符，然后继承ClassicConverter

首先定义类：SensitiveDataConverter 继承父类：MessageConverter

具体代码:

package eu.digient.sdk.util;

import ch.qos.logback.classic.pattern.MessageConverter;
import ch.qos.logback.classic.spi.ILoggingEvent;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;


public class SensitiveDataConverter extends MessageConverter {
    private final static String SENSITIVE_DEFAULT_DATA_KEYS = "fullName,address,ipAddress,nationalIdNumber,email,phoneNumber,birthDate,country,city,bankCard";
    private final static String SENSITIVE_DATA_MASKING_ENABLED = "true";

    @Override
    public String convert(ILoggingEvent event) {
        String originLogMsg = event.getFormattedMessage();
        return filterSensitiveInformation(originLogMsg);
    }

    public String filterSensitiveInformation(final String oriMsg) {
        String sensitiveDataMaskingEnabled = StringUtil.isEmptyString(System.getenv("SENSITIVE_DATA_MASKING_ENABLED")) ? SENSITIVE_DATA_MASKING_ENABLED : System.getenv("SENSITIVE_DATA_MASKING_ENABLED");
        String sensitiveDataKeys           = StringUtil.isEmptyString(System.getenv("SENSITIVE_DATA_KEYS")) ? SENSITIVE_DEFAULT_DATA_KEYS : System.getenv("SENSITIVE_DATA_KEYS");
        String tempMsg                     = oriMsg;

        if ("true".equals(sensitiveDataMaskingEnabled)) {
            if (sensitiveDataKeys != null && sensitiveDataKeys.length() > 0) {
                String[] keysArray = sensitiveDataKeys.split(",");

                for (String key : keysArray) {
                    int index = -1;
                    do {
                        index = tempMsg.indexOf(key, index + 1);
                        if (index != -1 &&
                            !Pattern.matches("[a-zA-Z0-9]", tempMsg.substring(index + key.length(), index + key.length() + 1)) &&
                            (index == 0 || !Pattern.matches("[a-zA-Z0-9]", tempMsg.substring(index - 1, index)))
                        ) {

                            int valueStart = getValueStartIndex(tempMsg, index + key.length());
                            int valueEnd   = getValueEndIndex(tempMsg, valueStart);

                            String subStr = tempMsg.substring(valueStart, valueEnd);
                            subStr = mask(subStr);
                            tempMsg = tempMsg.substring(0, valueStart) + subStr + tempMsg.substring(valueEnd);
                        }
                    } while (index != -1);
                }
            }
        }
        return tempMsg;
    }
    
    /**
     * Get the index of the first character of sensitive information
     */
    private int getValueStartIndex(String msg, int valueStart) {
        while (true) {
            if (valueStart == msg.length()) {
                break;
            }
            char ch = msg.charAt(valueStart);
            if (ch == ':' || ch == '=') {
                valueStart++;
                ch = msg.charAt(valueStart);
                if (ch == '"') {
                    valueStart++;
                }
                break;
            } else {
                valueStart++;
            }
        }
        return valueStart;
    }

    /**
     * Get the index of the last character of sensitive information
     */
    private int getValueEndIndex(String msg, int valueEnd) {
        while (true) {
            if (valueEnd == msg.length()) {
                break;
            }
            char ch = msg.charAt(valueEnd);
            if (ch == '"') {
                if (valueEnd + 1 == msg.length()) {
                    break;
                }
                char nextCh = msg.charAt(valueEnd + 1);
                if (nextCh == ';' || nextCh == ',') {
                    while (valueEnd > 0) {
                        char preCh = msg.charAt(valueEnd - 1);
                        if (preCh != '\\') {
                            break;
                        }
                        valueEnd--;
                    }
                    break;
                } else {
                    valueEnd++;
                }
            } else if (ch == ';' || ch == ',' || ch == '}') {
                break;
            } else {
                valueEnd++;
            }
        }
        return valueEnd;
    }

    /**
     * Mask sensitive information
     */
    private String mask(String submsg) {
        if (StringUtils.isBlank(submsg)) {
            return "";
        }
        return StringUtils.rightPad(StringUtils.left(submsg, 0), StringUtils.length(submsg), "*");
    }
}

在logback.xml 文件中加入（一定要加在appender标签的前面，最好就加在configuration标签的第一行，不然不会正确加载）

    <conversionRule conversionWord="msg" converterClass="eu.digient.sdk.util.SensitiveDataConverter"/>

converterClass写上你方法的路径

最后把几个test也放在这里：

package eu.digient.sdk.util

import org.junit.Test
import static org.junit.Assert.assertEquals

class SensitiveDataConverterTest {
    SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter()

    @Test
    void sensitiveDataTest() {
        assertEquals 'fullName =*******', sensitiveDataConverter.filterSensitiveInformation('fullName = tester')
        assertEquals 'fullName :*******', sensitiveDataConverter.filterSensitiveInformation('fullName : tester')
        assertEquals 'fullName=******,address=********', sensitiveDataConverter.filterSensitiveInformation('fullName=tester,address=some-add')
        assertEquals 'fullName:******,address:********', sensitiveDataConverter.filterSensitiveInformation('fullName:tester,address:some-add')
    }

    @Test
    void sensitiveDataConverter_withNomal() {
        def originLogMsg = '{nationalIdNumber=f88898b2677e62f1ad54b9e330c0a27e, currency=EUR, email=some@cn.com, fullName=%E5%BE%90%E5%BD%A6%E5%A8%9C, birthDate=2021-08-31}'
        def result = '{nationalIdNumber=********************************, currency=EUR, email=***********, fullName=***************************, birthDate=**********}'

        assertEquals result, sensitiveDataConverter.filterSensitiveInformation(originLogMsg)
    }

    @Test
    void sensitiveDataConverter_withJsonFormat() {
        def originLogMsg = '{"reason":"success ","result":{"jobid":"JH2131171027170837443588J6","fullName":"Tom","bankCard":"6226430106137525","nationalIdNumber":"130333198901192762","phoneNumber":"13210141605"}'
        def result = '{"reason":"success ","result":{"jobid":"JH2131171027170837443588J6","fullName":"***","bankCard":"****************","nationalIdNumber":"******************","phoneNumber":"************}'

        assertEquals result, sensitiveDataConverter.filterSensitiveInformation(originLogMsg)
    }

    @Test
    void sensitiveDataConverter_withJsonFormat2() {
        def originLogMsg = '{"body":"{\\"fullName\\":Tom,\\"address\\":\\someAdd\\}"}'
        def result = '{"body":"{\\"fullName\\":***,\\"address\\":*********}"}'

        assertEquals result, sensitiveDataConverter.filterSensitiveInformation(originLogMsg)
    }

    @Test
    void sensitiveDataConverter_withSemicolon() {
        def originLogMsg = '{nationalIdNumber=f88898b2677e62f1ad54b9e330c0a27el; currency=EUR; email=some@cn.com; fullName=%E5%BE%90%E5%BD%A6%E5%A8%9C; birthDate=2021-08-31}'
        def result = '{nationalIdNumber=*********************************; currency=EUR; email=***********; fullName=***************************; birthDate=**********}'

        assertEquals result, sensitiveDataConverter.filterSensitiveInformation(originLogMsg)
    }

    @Test
    void sensitiveDataConverter_withMoreChar() {
        def originLogMsg = '{nationalIdNumberOne=f88898b2677e62f1ad54b9e330c0a27el; currency=EUR; emailId=some@cn.com; somefullName=%E5%BE%90%E5%BD%A6%E5%A8%9C; birthDate=2021-08-31}'
        def result = '{nationalIdNumberOne=f88898b2677e62f1ad54b9e330c0a27el; currency=EUR; emailId=some@cn.com; somefullName=%E5%BE%90%E5%BD%A6%E5%A8%9C; birthDate=**********}'

        assertEquals result, sensitiveDataConverter.filterSensitiveInformation(originLogMsg)
    }
}