高效关键词匹配库效果对比分析

python

pip install flashtext

from flashtext import KeywordProcessor

kp = KeywordProcessor()
words = ["he", "she", "hers", "his", "share"] 
for word in words:
    kp.add_keyword(keyword=word)  # 新增关键词
kp.extract_keywords('ahishehersshare')
## out:  []

kp.extract_keywords('a his he hers share')  # 这里手动分词一下
## out：  ['his', 'he', 'hers', 'share']

github.com/ayoyu/flashtext

go get ggithub.com/ayoyu/flashtext

import (
	"github.com/ayoyu/flashtext"
	"fmt"
)

var (
	keys = []string{
		"he", "she", "hers", "his", "share",
	}
	key = "ahishershare"
)

func main() {
	// 创建一个关键词处理器，不区分大小写
	var flash = flashtext.NewFlashKeywords(false)

    for _, word := range keys {
        flash.Add(word)
	}
    matches := flash.Search(key)
    for _, match := range matches {
        fmt.Println("match:", match.Key)
        fmt.Println("Start:", match.Start)
        fmt.Println("End:", match.End)
        fmt.Println("---")
}
}
/*
		match: his
		Start: 1
		End: 3
		---
		match: he
		Start: 4
		End: 5
		---
		match: hers
		Start: 4
		End: 7
**/

github.com/code-innovator-zyx/flashtext

go get github.com/code-innovator-zyx/flashtext

import (
	"github.com/code-innovator-zyx/flashtext"
	"fmt"
)

var (
	keys = []string{
		"he", "she", "hers", "his", "share",
	}
	key = "ahishershare"
)

func main() {
	// 创建一个关键词处理器，不区分大小写
	kp := flashtext.NewKeywordProcessor(false)

	// 添加关键词
	kp.AddKeywordsFromList(keys).Build()

	// 提取关键词
	matches := kp.ExtractKeywords(key)

	// 打印匹配结果
	for _, match := range matches {
		fmt.Println("Match:", match.Match)
		fmt.Println("Start:", match.Start)
		fmt.Println("End:", match.End)
		fmt.Println("---")
	}
}

/**
			his
			Start: 1
			End: 3
			---
			she
			Start: 3
			End: 5
			---
			he
			Start: 4
			End: 5
			---
			hers
			Start: 4
			End: 7
			---
			share
			Start: 7
			End: 11
			---
*/

所以 github.com/code-innovator-zyx/flashtext 库目前使用的标准ac自动机，匹配的效果是目前最好的一个库，基于大量关键词，长文本匹配无疑ac自动机算法性能无需质疑。但是可惜的是暂不支持数据清洗功能，因为是多模式匹配，并且内部构建了失败指针，如果进行数据清洗，会有重复清洗的情况，最后输出效果可能与预期的大相径庭，可能后面会支持吧