问题
‘ChatGLMTokenizer’ object has no attribute ‘sp_tokenizer’
解决方案
将self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)移动到super().init()前面
self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
super().__init__(
do_lower_case=do_lower_case,
remove_space=remove_space,
padding_side=padding_side,
bos_token=bos_token,
eos_token=eos_token,
end_token=end_token,
mask_token=mask_token,
gmask_token=gmask_token,
pad_token=pad_token,
unk_token=unk_token,
num_image_tokens=num_image_tokens,
**kwargs
)
self.do_lower_case = do_lower_case
self.remove_space = remove_space
self.vocab_file = vocab_file
self.bos_token = bos_token
self.eos_token = eos_token
self.end_token = end_token
self.mask_token = mask_token
self.gmask_token = gmask_token
#self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)