【NLP】基于字典的分词——最大前向匹配、最大后向匹配、双向匹配

最大前向匹配

def forward_match(dic,text):
    i = 0
    word_list = []
    while i<len(text):
        word = text[i]
        for j in range(i+1,len(text)+1):
            long_word = text[i:j]
            if long_word in dic and len(long_word) > len(word):
                word = long_word
        word_list.append(word)
        i += len(word)

    return word_list

dic = {"效果","研究","口红","中国","进口","红酒","中国进口"}
text = "中国进口红酒"
#word_list = forward_match(dic,text)
#print(word_list)

最大后向匹配

def backward_match(dic,text):
    i = len(text)-1
    word_list = []
    while i>-1:
        word = text[i]
        for j in range(len(text)-2,-1,-1):
            long_word = text[j:i+1]
            if long_word in dic and len(long_word) > len(word):
                word = long_word
        word_list.append(word)
        i -= len(word)

    return word_list

dic = {"效果","研究","口红","中国","进口","红酒","中国进口","研究生","起源","生命"}
text = "研究生命起源"

print(forward_match(dic,text))
print(backward_match(dic,text))

# 判断词典单字数

```python
def count_single_char(word_list):
    count = 0
    for i in word_list:
        if len(i)==1:
            count += 1
    return count 

print(count_single_char(forward_match(dic,text)))

# 双向最大匹配

```python
def doubleward_match(dic,text):
    
    f = forward_match(dic,text)
    b = backward_match(dic,text)
    if len(f)>len(ba):
        return b
    elif len(f)>len(b):
        return f
    else:
        if count_single_char(f)<count_single_char(b)
            return f
        else:
            return b