聊天机器人之文本分词

聊天机器人之文本分词

1. 准备词典和停用词

1.1 准备词典

1.2 准备停用词

stopwords = set([i.strip() for i in open(config.stopwords_path).readlines()])

关于结巴分词的详细代码操作，查看博文：https://blog.csdn.net/weixin_44799217/article/details/115244829

2. 准备按照单个字切分句子的方法

import re


def cut_by_word(sentence):
    # 对中文按照字进行处理，对英文不分为字母
    sentence = re.sub("\s+", " ", sentence)  # 匹配空白，即 空格，tab键
    sentence = sentence.strip()
    result = []
    temp = ""
    for word in sentence:
        if word.lower() in letters:
            temp += word.lower()
        else:
            if temp != "":  # 不是字母
                result.append(temp)
                temp = ""
            if word.strip() in filters:  # 标点符号
                continue
            else:  # 是单个字
                result.append(word)
    if temp != "":  # 最后的temp中包含字母
        result.append(temp)
    return result

3. 完成分词方法的封装

lib 下创建cut_sentence.py文件，完成分词方法的构建

import logging
import jieba
import jieba.posseg as psg  # 分词并返回词性
import config
import re
import string

# 关闭jieba log输出
jieba.setLogLevel(logging.INFO)  # 这行代码使程序运行时的4行红字去掉
# 加载词典
jieba.load_userdict(config.keywords_path)
# 单字分割，英文部分
letters = string.ascii_lowercase
# 单字分割 去除的标点
filters = [",", "-", ".", " "]
# 停用词
stopwords = set([i.strip() for i in open(config.stopwords_path).readlines()])


def cut(sentence, by_word=False, use_stopwords=False, with_sg=False):
    """
    
    :param sentence: str句子
    :param by_word: 是否按照单个字分词
    :param use_stopwords: 是否使用停用词
    :param with_sg: 是否返回词性
    :return: 
    """
    assert by_word != True or with_sg != True, "根据word切分时候无法返回词性"
    if by_word:
        return _cut_by_word(sentence)
    else:
        ret = psg.lcut(sentence)
        if use_stopwords:
            ret = [(i.word, i.flag) for i in ret if i.word not in stopwords]
        if not with_sg:
            ret = [i.word for i in ret]
        return ret


def _cut_by_word(sentence):
    # 对中文按照字进行处理，对英文不分为字母
    sentence = re.sub("\s+", " ", sentence)
    sentence = sentence.strip()
    result = []
    temp = ""
    for word in sentence:
        if word.lower() in letters:
            temp += word.lower()
        else:
            if temp != "":  # 不是字母
                result.append(temp)
                temp = ""
            if word.strip() in filters:  # 标点符号
                continue
            else:  # 是单个字
                result.append(word)
    if temp != "":  # 最后的temp中包含字母
        result.append(temp)
    return result