如何设计高效的聊天过滤词算法？

关于聊天过滤词算法，一直困扰着我，了解到很多算法，比如：KMP, 正则循环匹配等，然后在http://www.dewen.org/q/41/%E5%A6%82%E4%BD%95%E8%AE%BE%E8%AE%A1%E9%AB%98%E6%95%88%E7%9A%84%E8%81%8A%E5%A4%A9%E8%BF%87%E6%BB%A4%E8%AF%8D%E7%AE%97%E6%B3%95%EF%BC%9F看到了一篇文章，现摘要几种相对好的答案，以备不时之需。

1trie树算法

我们的解决方法是用构造一个tire树。 每个节点都存储0- 256个字符。

用脏词字典来构造这个树。

具体实现代码如下：

namespace KGame
{
class WordFilter
{
public:
    WordFilter() {}

    ~WordFilter() 
    {
        Clean(&m_Filter);
    }

    void AddWord(const char* word)
    {
        UInt32 len = (UInt32)strlen(word);
        Filter* filter = &m_Filter;

        for (UInt32 i = 0; i < len; i++)
        {
            unsigned char c = word[i];
            if (i == len - 1)
            {
                filter->m_NodeArray[c].m_Flag |= FilterNode::NODE_IS_END;
                break;
            }
            else
            {
                filter->m_NodeArray[c].m_Flag |= FilterNode::NODE_HAS_NEXT;
            }

            if (filter->m_NodeArray[c].m_NextFilter == NULL)
            {
                Filter* tmpFilter = XNEW (Filter)();
                filter->m_NodeArray[c].m_NextFilter = tmpFilter;
            }

            filter = (Filter *)filter->m_NodeArray[c].m_NextFilter;
        }
    }

    void AddWords(const std::set<std::string>& wordList)
    {
        for (std::set<std::string>::const_iterator it = wordList.begin();
            it != wordList.end(); it++)
        {
            AddWord(it->c_str());
        }
    }

    void AddWords(const std::vector<std::string>& wordList)
    {
        for (std::vector<std::string>::const_iterator it = wordList.begin();
            it != wordList.end(); it++)
        {
            AddWord(it->c_str());
        }
    }

    void AddWords(const KGame::Set<std::string>& worldList)
    {
        for (KGame::Set<std::string>::Iter* iter = worldList.Begin();
            iter != worldList.End(); iter = worldList.Next(iter))
        {
            AddWord(iter->m_Value.c_str());
        }
    }

    Int32 Check(const char* str)
    {
        Filter* filter = NULL;
        for (Int32 i = 0; i < (int)strlen(str) - 1; i++)
        {
            filter = &m_Filter;
            for (UInt32 j = i; j < strlen(str); j++)
            {
                unsigned char c = str[j]; 
                if ((c >= 'A' && c <= 'Z'))
                {
                    c += 32;
                }

                if (filter->m_NodeArray[c].m_Flag == FilterNode::NODE_IS_NULL)
                {
                    break;
                }
                else if (filter->m_NodeArray[c].m_Flag & FilterNode::NODE_IS_END)
                {
                    return i;
                }
                else // NODE_HAS_NEXT
                {
                    filter = (Filter*)filter->m_NodeArray[c].m_NextFilter;
                }
            }
        }
        return -1;
    }

    void CheckAndModify(char* str, const char replace = '*')
    {
        Filter* filter = NULL;
        for (Int32 i = 0; i < (int)strlen(str) - 1; i++)
        {
            filter = &m_Filter;
            for (UInt32 j = i; j < strlen(str); j++)
            {
                unsigned char c = str[j]; 
                if ((c >= 'A' && c <= 'Z'))
                {
                    c += 32;
                }

                if (filter->m_NodeArray[c].m_Flag == FilterNode::NODE_IS_NULL)
                {
                    break;
                }
                else if (filter->m_NodeArray[c].m_Flag & FilterNode::NODE_IS_END)
                {
                    for (UInt32 k = i; k <= j; k++)
                    {
                        str[k] = replace;
                    }

                    if (filter->m_NodeArray[c].m_Flag & FilterNode::NODE_HAS_NEXT)
                    {
                        filter = (Filter*)filter->m_NodeArray[c].m_NextFilter;
                    }
                    else
                    {
                        continue;
                    }
                }
                else // NODE_HAS_NEXT
                {
                    filter = (Filter*)filter->m_NodeArray[c].m_NextFilter;
                }
            }
        }
    }

    void CheckAndModify(std::string& str, const char replace = '*')
    {
        Filter* filter = NULL;
        for (Int32 i = 0; i < (int)str.size() - 1; i++)
        {
            filter = &m_Filter;
            for (UInt32 j = i; j < str.size(); j++)
            {
                unsigned char c = str[j]; 
                if ((c >= 'A' && c <= 'Z'))
                {
                    c += 32;
                }
                if (filter->m_NodeArray[c].m_Flag == FilterNode::NODE_IS_NULL)
                {
                    break;
                }
                else if (filter->m_NodeArray[c].m_Flag & FilterNode::NODE_IS_END)
                {
                    for (UInt32 k = i; k <= j; k++)
                    {
                        str[k] = replace;
                    }

                    if (filter->m_NodeArray[c].m_Flag & FilterNode::NODE_HAS_NEXT)
                    {
                        filter = (Filter*)filter->m_NodeArray[c].m_NextFilter;
                    }
                    else
                    {
                        continue;
                    }
                }
                else // NODE_HAS_NEXT
                {
                    filter = (Filter*)filter->m_NodeArray[c].m_NextFilter;
                }
            }
        }
    }

private:

    struct FilterNode
    {
        char m_Flag;
        void* m_NextFilter;

        enum Flag
        {
            NODE_IS_NULL          = 0x00,
            NODE_HAS_NEXT         = 0x01,
            NODE_IS_END           = 0x10,
        };
        FilterNode() : m_Flag(NODE_IS_NULL), m_NextFilter(NULL) {}
    };

    struct Filter
    {
        FilterNode m_NodeArray[256];
    } m_Filter;

    void Clean(Filter* filter)
    {
        for (UInt32 i = 0; i < 256; i++)
        {
            if (filter->m_NodeArray[i].m_NextFilter)
            {
                Clean((Filter *)filter->m_NodeArray[i].m_NextFilter);
                XDELETE((Filter*)filter->m_NodeArray[i].m_NextFilter);
            }
        }
    }
};
} // namespace KGame

2.基于KMP算法

聊天过滤词算法的解决思路
提高过滤的算法个人认为主要从两个方面考虑：（1）尽量减少内存、IO的次数。（2）增加串内查找的速度。
基于这两点我想采用连续的内存片，可以减少内存地址跳跃的次数，采用静态的内存这就解决了（1）的问题，第二点是增加串内查找的速度，这个比较公认的事KMP算法

class WordFilter
{
public:
WordFilter();
~WordFilter();

void Init();
void FilterWord(string& word);
int  Index_KMP(const char* S, const char* T, int pos);

private:
std::set<string> m_storage;
const char** m_words;
uint32 m_count;
};

WordFilter::WordFilter()
{
m_words = NULL;
m_count = 0;
}

WordFilter::~WordFilter()
{
if(m_words) {
free(m_words);
}
}

void WordFilter::Init()
{
// 把所有屏蔽词都放到m_storage里
m_count = m_storage.size();
if(m_count) {
m_words = (const char**)malloc(sizeof(char*)*m_count);
std::set<string>::iterator ptr;
int i = 0;
for(ptr = m_storage.begin(); ptr != m_storage.end(); ++ptr,i++) {
m_words[i] = ptr->c_str();
}
}
}

static inline void _filterWord(char* word, const char* lowerWord, const char* oldstr)
{
int len = strlen(oldstr);
const char* tmp;
memset(word, '*', len);
word += len;
lowerWord += len;

while((tmp = Index_KMP(lowerWord, oldstr)) != NULL) {
word += (tmp-lowerWord);
memset(word, '*', len);
word += len;
lowerWord = tmp + len;
}
}

void WordFilter::FilterWord(string& word)
{
string tmp(word);
str_tolower(tmp);
const char** p = (const char**)m_words;
const char* dest;
for(uint32 i=0; i<m_count; i++, p++) {
dest = Index_KMP(tmp.c_str(), *p, 0);
if(dest) {
_filterWord((char*)(word.c_str() + (dest-tmp.c_str())), dest, *p);
}
}
}

int WordFilter::Index_KMP(const char* S, const char* T, int pos){    
i=pos;   j=1;  
while(i <= S[0] && j<= T[0]){  
if(j == 0 || S[i] == T[j]) { ++i; ++j; }
else   j = next[j];
}  

if(j>T[0])   
return   i-T[0]; 
else   
return   0;  
}

以上两种方法相对比较好一点。以做参考。