import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* User: eternity
* Date: 2014/8/11
* Time: 16:17
* User:zxt 2019/7/22 15:00 up 修复拦截大小写Bug
* 敏感词检测类
* 敏感词检测初始化规则:
* 将敏感词从词库载入,按照2字、3字、4字、5字等字数各生成一个敏感词哈希表。
* 在将这些哈希表组成一个数组banWordsList,数组下标表示该敏感词表字数
* banWordsList[2] = {某马:true,屏蔽:true,啦啦:true};
* banWordsList[3] = {某个马:true,三个字:true,啦啦啦:true,小广告:true};
* banWordsList[4] = {某个坏银:true,四个字符:true,哈哈哈哈:true,就爱凤姐:true};
* banWordsList[5] = {某个大法好:true,五个敏感字:true};
* 根据上面几组组敏感词,自动生成以下索引
* 生成规则为,索引名是敏感词第一个字,值是一个int
* 该int的规则为,该int转换成二进制时,第i位为1表示上面4表存在长度为i的敏感词,否则不存在长度为i的敏感词(10000)
* wordIndex = {二:0x04,三:0x08,四:0x10,五:0x20,某:0x3c,啦:0x0c,哈:0x10,小:0x08,就:0x10};
*
* 检查规则如下:
* 1,逐字检验,是否该字在wordIndex索引表中。
* 2,如果不在表中,继续检验
* 3,如果在表中,根据索引表该键的值,取此字以及此字后的若干字检验详细表banWordsList[索引词长]。
*
* 检验例子
* 有一段如下文字,检验其是否包含敏感词:
* “我就打小广告,气死版主”
* ——检测“我”
* |-不在索引表
* ——检测“就”
* |-在索引表
* |-“就”的索引值是0x10,表示有4字以“就”开头的敏感词
* |-取“就”和后面的字共4个,组成“就打小广”
* |-查4字敏感词表,没有这项,继续
* ——检测“打”
* |-不在索引表
* ——检测“小”
* |-在索引表
* |-索引值是0x08,表示有3字长度的敏感词
* |-取“小”和“小”后面的字,共3个字组成一个词“小广告”
* |-“小广告”在3字敏感词中,此帖包含敏感词,禁止发布
*/
public class NickWordsUtil {
// public Logger logger = Logger.getLogger(this.getClass());
public static final int WORDS_MAX_LENGTH = 30;
//敏感词列表
@SuppressWarnings("rawtypes")
public static Map[] banWordsList = null;
//敏感词索引
public static Map<String, Integer> wordIndex = new HashMap<String, Integer>();
/*
* 初始化敏感词库
*/
@SuppressWarnings("unchecked")
public static void initBanWordsList(List<String> words) {
if (banWordsList == null) {
banWordsList = new Map[WORDS_MAX_LENGTH];
for (int i = 0; i < banWordsList.length; i++) {
banWordsList[i] = new HashMap<String, String>();
}
}
for (String w : words) {
if (StringUtils.isNotBlank(w)) {
//将敏感词按长度存入map
banWordsList[w.length()].put(w.toLowerCase(), "");
Integer index = wordIndex.get(w.substring(0, 1));
//生成敏感词索引,存入map
if (index == null) {
index = 0;
}
int x = (int) Math.pow(2, w.length());
index = (index | x);
wordIndex.put(w.substring(0, 1).toLowerCase(), index);
}
}
}
/**
* 检索敏感词
* @param content
* @return
*/
public static List<String> searchBanWords(List<String> words,String content) {
if (banWordsList == null) {
initBanWordsList(words);
}
List<String> result = new ArrayList<String>();
for (int i = 0; i < content.length(); i++) {
Integer index = wordIndex.get(content.substring(i, i + 1));
int p = 0;
while ((index != null) && (index > 0)) {
p++;
index = index >> 1;
String sub = "";
if ((i + p) < (content.length())) {
sub = content.substring(i, i + p);
} else {
sub = content.substring(i);
}
if (((index % 2) == 1) && banWordsList[p].containsKey(sub)) {
result.add(content.substring(i, i + p));
}
}
}
return result;
}
/**
* 检索敏感词
* @param content
* @return
*/
public static boolean isBanWords(List<String> words,String content) {
boolean b = false;
if (banWordsList == null) {
initBanWordsList(words);
}
OUT:
for (int i = 0; i < content.length(); i++) {
Integer index = wordIndex.get(content.substring(i, i + 1));
int p = 0;
while ((index != null) && (index > 0)) {
p++;
index = index >> 1;
String sub = "";
if ((i + p) < (content.length())) {
sub = content.substring(i, i + p);
} else {
sub = content.substring(i);
}
if (((index % 2) == 1) && banWordsList[p].containsKey(sub)) {
b = true;
break OUT;
}
}
}
return b;
}
/**
* @function 检索敏感词并返回所有违规的敏感词汇
* @author zhaoxutao
* @param content(自动转化为小写,请勿重复操作)
* @since @param : content 将自动转化为小写,请勿重复操作;
* @return List<String> :违规词组
*
*/
public static List<String> isBanWordsList(List<String> words,String content) {
if (banWordsList == null) {
initBanWordsList(words);
}
List<String> list = new ArrayList<>();
//转小写校对
String lowerCaseContent = content.toLowerCase();
for (int i = 0; i < lowerCaseContent.length(); i++) {
//词汇长度索引获取
Integer index = wordIndex.get(lowerCaseContent.substring(i, i + 1));
int p = 0;
while ((index != null) && (index > 0)) {
p++;
index = index >> 1;
String sub = "";
//记录当前词汇在源词汇中的索引地址
int stringIndex = 0;
if ((i + p) < (lowerCaseContent.length())) {
sub = lowerCaseContent.substring(i, i + p);
stringIndex = i + p;
} else {
sub = lowerCaseContent.substring(i);
stringIndex = lowerCaseContent.length() - 1;
}
if (((index % 2) == 1) && banWordsList[p].containsKey(sub)) {
//根据词汇索引在源词汇中提取源词汇
list.add(content.substring(i, stringIndex));
break ;
}
}
}
return list;
}
/**
* @function 重置内存
* @author zhaoxutao
* @date 2019-8-10 10:03
* @param content
* @return
*/
public static void recover() {
if (banWordsList != null) {
banWordsList = null;
wordIndex = new HashMap<String, Integer>();
}
}
敏感词汇检测及返回敏感源词汇
最新推荐文章于 2024-02-28 00:57:57 发布