# tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
改成这个legacy=False, use_fast=False:
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, legacy=False, use_fast=False)
# tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
改成这个legacy=False, use_fast=False:
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, legacy=False, use_fast=False)