上午没问题,下午pull了新代码,就有了报错。。
发现是transformers版本问题。但。。其实我都默认安的是最新版本。。
也许这就是人生吧。。
报错:
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 213, in build_tokenizer
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
tokenizer = _Qwen2Tokenizer(args.load, args.extra_vocab_size)
File "/Pai-Megatron-Patch/megatron_patch/tokenizer/__init__.py", line 174, in __init__
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
self.tokenizer = AutoTokenizer.from_pretrained(
File "/usr/local/lib/python3.10/dist-packages/transformers/models/auto/tokenization_auto.py", line 784, in from_pretrained
raise ValueError(raise ValueError(
ValueError ValueErrorraise ValueError(: :
Tokenizer class Qwen2Tokenizer does not exist or is not currently imported.Tokenizer class Qwen2Tokenizer does not exist or is not currently imported.
raise ValueError(raise ValueError( ValueError
raise ValueError(
解决方案:
原本的:
pip install -U transformers -i https://pypi.doubanio.com/simple/
改成:
pip install transformers==4.40 -i https://mirrors.aliyun.com/pypi/simple/
Successfully installed huggingface-hub-0.23.3 safetensors-0.4.3 tokenizers-0.19.1 transformers-4.41.2