Bert两种Fast Tokenizer输出比较

BertTokenizer是slow tokenizer，没有offset_mapping功能，所以要手动使用Fast。以下两种tokenizer输出可以一致，其中BertTokenizer也可以换成AutoTokenizer。

from tokenizers import BertWordPieceTokenizer
from transformers import BertTokenizerFast


MODEL_PATH = "/bert-base-uncased"

context="hello to World"

"""fast tokenizer of BertWordPieceTokenizer"""
tokenizer = BertWordPieceTokenizer(MODEL_PATH + "/vocab.txt", lowercase=True)
tokenized_context = tokenizer.encode(context)
print(tokenized_context)
print(tokenized_context.ids)
print(tokenized_context.type_ids)
print(tokenized_context.tokens)
print(tokenized_context.offsets)
print(tokenized_context.attention_mask)
"""fast tokenizer of BertTokenizerFast"""
tokenizer = BertTokenizerFast.from_pretrained(MODEL_PATH)
tokens = tokenizer.tokenize(context, add_special_tokens=True)
tokenized_context = tokenizer.encode_plus(context, return_offsets_mapping=True)
print(tokenized_context)
print(tokenized_context.input_ids)
print(tokenized_context.token_type_ids)
print(tokens)
print(tokenized_context.offset_mapping)
print(tokenized_context.attention_mask)

输出

Encoding(num_tokens=5, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])
[101, 7592, 2000, 2088, 102]
[0, 0, 0, 0, 0]
['[CLS]', 'hello', 'to', 'world', '[SEP]']
[(0, 0), (0, 5), (6, 8), (9, 14), (0, 0)]
[1, 1, 1, 1, 1]
{'input_ids': [101, 7592, 2000, 2088, 102], 'token_type_ids': [0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1], 'offset_mapping': [(0, 0), (0, 5), (6, 8), (9, 14), (0, 0)]}
[101, 7592, 2000, 2088, 102]
[0, 0, 0, 0, 0]
['[CLS]', 'hello', 'to', 'world', '[SEP]']
[(0, 0), (0, 5), (6, 8), (9, 14), (0, 0)]
[1, 1, 1, 1, 1]