Java使用DFA算法处理敏感词汇

1. 初始化敏感词库，将敏感词加入到HashMap中，构建DFA算法模型

package com.datago.common.utils.sensitive;


import java.util.*;

/**
 * @ProjectName innovate  初始化敏感词库，将敏感词加入到HashMap中，构建DFA算法模型
 * @Package com.datago.common.utils.sensitive
 * @Name SensitiveWordInit
 * @Author HB
 * @Date 2022/1/25 18:12
 * @Version 1.0
 */

public class SensitiveWordInit {
    @SuppressWarnings("rawtypes")
    public static HashMap sensitiveWordMap;

    public SensitiveWordInit() {
        super();
    }

    /**
     * 初始化词库
     *
     * @param datas 敏感词集合
     * @return
     */
    public static HashMap init(String datas) {
        addSensitiveWord(datas);
        return sensitiveWordMap;
    }

    private static void addSensitiveWord(String word) {
        sensitiveWordMap = new HashMap(word.length());
        Map<String, Object> now = null;
        Map now2 = null;
            now2 = sensitiveWordMap;
            for (int i = 0; i < word.length(); i++) {
                char key_word = word.charAt(i);
                Object obj = now2.get(key_word);
                if (obj != null) { //存在
                    now2 = (Map) obj;
                } else { //不存在
                    now = new HashMap<>();
                    now.put("isEnd", "0");
                    now2.put(key_word, now);
                    now2 = now;
                }
                if (i == word.length() - 1) {
                    now2.put("isEnd", "1");
                }
            }
    }

    /**
     * 获取内容中的敏感词
     *
     * @param text      内容
     * @param matchType 匹配规则 1=不最佳匹配，2=最佳匹配
     * @return
     */
    public static List<String> getSensitiveWord(String text, int matchType) {
        List<String> words = new ArrayList<String>();
        Map now = sensitiveWordMap;
        int count = 0;  //初始化敏感词长度
        int start = 0; //标志敏感词开始的下标
        for (int i = 0; i < text.length(); i++) {
            char key = text.charAt(i);
            now = (Map) now.get(key);
            if (now != null) { //存在
                count++;
                if (count == 1) {
                    start = i;
                }
                if ("1".equals(now.get("isEnd"))) { //敏感词结束
                    now = sensitiveWordMap; //重新获取敏感词库
                    words.add(text.substring(start, start + count)); //取出敏感词，添加到集合
                    count = 0; //初始化敏感词长度
                }
            } else { //不存在
                now = sensitiveWordMap;//重新获取敏感词库
                if (count == 1 && matchType == 1) { //不最佳匹配
                    count = 0;
                } else if (count == 1 && matchType == 2) { //最佳匹配
                    words.add(text.substring(start, start + count));
                    count = 0;
                }
            }
        }
        return words;
    }
}

2. 敏感词过滤

package com.datago.common.utils.sensitive;

import com.datago.common.core.redis.RedisCache;
import com.datago.common.utils.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.*;

/**
 * @ProjectName innovate 敏感词过滤
 * @Package com.datago.common.utils.sensitive
 * @Name SensitivewordFilter
 * @Author HB
 * @Date 2022/1/25 18:14
 * @Version 1.0
 */

@Component
public class SensitivewordFilter {


    private static RedisCache redisCache;

    @Autowired
    public void setRedisCache(RedisCache redisCache) {
        SensitivewordFilter.redisCache = redisCache;
    }

    @SuppressWarnings("rawtypes")
    private static Map sensitiveWordMap = null;


    public static void initSensitiveWord(String datas) {
        sensitiveWordMap = SensitiveWordInit.init(datas);
    }

    /**
     * 替换敏感字字符
     *
     * @param txt
     * @param matchType
     * @param replaceChar 替换字符，默认*
     * @author HB
     * @version 1.0
     */
    public static String replaceSensitiveWord(String datas, String txt, int matchType, String replaceChar) {
        if (sensitiveWordMap == null) {
            initSensitiveWord(datas);
        }
        String resultTxt = txt;
        //matchType = 1;      //最小匹配规则
        //matchType= 2;      //最大匹配规则
        List<String> set = SensitiveWordInit.getSensitiveWord(txt, matchType);     //获取所有的敏感词
        Iterator<String> iterator = set.iterator();
        String word = null;
        String replaceString = null;
        while (iterator.hasNext()) {
            word = iterator.next();
            replaceString = getReplaceChars(replaceChar, word.length());
            resultTxt = resultTxt.replaceAll(word, replaceString);
        }
        return resultTxt;
    }

    /**
     * 获取替换字符串
     *
     * @param replaceChar
     * @param length
     * @return
     * @author HB
     * @version 1.0
     */
    private static String getReplaceChars(String replaceChar, int length) {
        String resultReplace = replaceChar;
        if (length > 6) {
            length = 6;
        }
        for (int i = 1; i < length; i++) {
            resultReplace += replaceChar;
        }
        return resultReplace;
    }


    /**
     * 过滤敏感词汇
     *
     * @param sensitiveTxt 输入数据
     * @return com.datago.common.core.domain.AjaxResult
     * @Author HB
     * @Date 2022/1/27 10:03
     **/
    public static String filterSensitive(String sensitiveTxt) {
        //从缓存中提取数据敏感词汇
        Map<String, String> datas = redisCache.getCacheObject("treeSensitive");
        //替换敏感词汇
        String updateTxt = null;
        for (Map.Entry<String, String> entry : datas.entrySet()) {
            SensitivewordFilter.initSensitiveWord(entry.getKey());
            if (StringUtils.isNotEmpty(updateTxt)) {
                updateTxt = replaceSensitiveWord(entry.getKey(), updateTxt, 1, entry.getValue());
            } else {
                updateTxt = replaceSensitiveWord(entry.getKey(), sensitiveTxt, 1, entry.getValue());
            }
        }
        return updateTxt;
    }

}

3.应用

   /**
     * 过滤datago_sensitive敏感词汇
     * sensitiveTxt  传参
     */
    @Log(title = "过滤敏感词汇")
    @GetMapping("/filterSensitive/{sensitiveTxt}")
    public AjaxResult filterSensitive(@PathVariable(value = "sensitiveTxt") String sensitiveTxt) {
        String s = SensitivewordFilter.filterSensitive(sensitiveTxt);
        return AjaxResult.success(s);
    }

4.参考文献

https://www.hutool.cn/docs/#/dfa/DFA%E6%9F%A5%E6%89%BE