实现word2sequence

coding=utf-8

“”"
author:lei
function: 实现的是构建词典，实现方法把句子转换为数字序列并将其翻转
“”"

class Word2Sequence(object):
UNK_TAG = “UNK”
PAD_TAG = “PAD”

UNK = 0
PAD = 1

def __init__(self):
    self.dict = {
        self.UNK_TAG: self.UNK,
        self.PAD_TAG: self.PAD
    }

    self.count = {}  # 统计词频

def fit(self, sentence):
    """
        把单个句子保存到dict中
    :param sentence: [word1,word2,word3...]
    :return:
    """
    for word in sentence:
        # 判断当前字典中是否有这个词，有则在原先词频的基础上+1，没有则0+1
        self.count[word] = self.count.get(word, 0) + 1

# 限制词频最小值和最大值
def build_vocab(self, min=5, max=None, max_features=None):
    """
        生成词典
    :param min: 最小出现的次数
    :param max_feature: 最大的次数
    :param max_features: 一共保留多少个词语
    :return:
    """
    # 删除count中词频小于min的word
    if min is not None:
        self.count = {word: value for word, value in self.count.items() if value > min}
    # 删除词语大于max的值
    if max is not None:
        self.count = {word: value for word, value in self.count.items() if value < max}
    # 限制保留的词语数
    if max_features is not None:
        # 对所有单词出现的频数进行排序，保留前max_features
        temp = sorted(self.count.items(), key=lambda x: x[-1], reverse=True)[:max_features]
        # 将temp转换为字典
        self.count = dict(temp)

    for word in self.count:
        # 从二开始对每个单词匹配其对应的数字
        self.dict[word] = len(self.dict)

    # 得到一个翻转的dict字典
    self.inverse_dict = dict(zip(self.dict.values(), self.dict.keys()))

def transform(self, sentence, max_len=None):
    """
        把句子转换为序列
    :param sentence: [word1, word2,...]
    :param max_len: int 对句子进行填充或者裁剪
    :return:
    """
    # for word in sentence:
    #     # 将每个句子中的单词转换为对应的数字，如果没有则为UNK
    #     self.dict.get(word, self.UNK)

    if max_len is not None:
        if max_len > len(sentence):
            # 填充
            sentence = sentence + [self.PAD_TAG] * (max_len - len(sentence))
        if max_len < len(sentence):
            # 裁剪
            sentence = sentence[:max_len]

    return [self.dict.get(word, self.UNK) for word in sentence]

def inverse_transform(self, indices):
    """
        把序列转换为句子
    :param indices: [1,2,3,4...]
    :return:
    """
    return [self.inverse_dict.get(idx) for idx in indices]

def __len__(self):
    return len(self.dict)

if name == ‘main’:
ws = Word2Sequence()
ws.fit([“我”, “是”, “谁”])
ws.fit([“我”, “是”, “我”])
ws.build_vocab(min=0)
print(ws.dict)

ret = ws.transform(["我", "爱", "北京"], max_len=10)
print(ret)
ret = ws.inverse_transform(ret)
print(ret)