问题描述:
使用MindSpore1.0.0版本进行模型训练时,在model.train()后报错如下:
----> 1 model.train(config.epochs, train_dataset, callbacks=cb, dataset_sink_mode=False)
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/train/model.py in train(self, epoch, train_dataset, callbacks, dataset_sink_mode, sink_size)
562 callbacks=callbacks,
563 dataset_sink_mode=dataset_sink_mode,
--> 564 sink_size=sink_size)
565
566 def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None):
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/train/model.py in _train(self, epoch, train_dataset, callbacks, dataset_sink_mode, sink_size)
366 with _CallbackManager(callbacks) as list_callback:
367 if not dataset_sink_mode:
--> 368 self._train_process(epoch, train_dataset, list_callback, cb_params)
369 elif context.get_context("mode") == context.PYNATIVE_MODE or context.get_context("device_target") == "CPU":
370 logger.warning("The pynative mode and CPU cannot support dataset sink mode currently."
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/train/model.py in _train_process(self, epoch, train_dataset, list_callback, cb_params)
461 phase='train',
462 dataset=train_dataset,
--> 463 dataset_sink_mode=False)
464 cb_params.cur_step_num = 0
465 run_context = RunContext(cb_params)
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/train/model.py in _exec_preprocess(self, network, is_train, phase, dataset, dataset_sink_mode, sink_size, epoch_num)
255 if dataset_sink_mode and not is_train:
256 dataset.__loop_size__ = 1
--> 257 dataset_helper = DatasetHelper(dataset, dataset_sink_mode, sink_size, epoch_num)
258
259 if dataset_sink_mode:
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/train/dataset_helper.py in __init__(self, dataset, dataset_sink_mode, sink_size, epoch_num)
146 else:
147 iterclass = _DatasetIterNormal
--> 148 self.iter = iterclass(dataset)
149
150 def __iter__(self):
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/train/dataset_helper.py in __init__(self, dataset)
295 self.device_num = _get_device_num()
296 self.global_rank = _get_global_rank()
--> 297 self.iter = self.dataset.create_tuple_iterator()
298
299 def __iter__(self):
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/dataset/engine/validators.py in new_method(self, *args, **kwargs)
285 nreq_param_bool = ['output_numpy']
286 validate_dataset_param_value(nreq_param_bool, param_dict, bool)
--> 287 return method(self, *args, **kwargs)
288
289 return new_method
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/dataset/engine/datasets.py in create_tuple_iterator(self, columns, num_epochs, output_numpy)
1186 if self._noop_mode():
1187 return DummyIterator(self, 'tuple')
-> 1188 return TupleIterator(self, columns, num_epochs, output_numpy)
1189
1190 @check_iterator
~/.virtualenvs/basenv/lib/python3.7/site-packages/mindspore/dataset/engine/iterators.py in __init__(self, dataset, columns, num_epochs, output_numpy)
358 dataset = dataset.project(columns)
359 super().__init__(dataset, num_epochs, output_numpy)
--> 360 self.depipeline.LaunchTreeExec()
361
362 def __iter__(self):
RuntimeError: Thread ID 140231386838784 Unexpected error. Fail to find size of file
Line of code : 267
File : /home/jenkins/agent-working-dir/workspace/Compile_CPU_X86_Ubuntu/mindspore/mindspore/ccsrc/minddata/dataset/core/tensor.cc
请问这是什么原因造成的?解决方法是什么?
解答:
应该是将本地文件读取变成Tensor的时候进行出错了,需要检查一下这个文件的合法性,mindspore1.0的不能打印这个路径,可以安装一下新版本的mindspore(1.7)打印一下这个文件路径。