问题
TypeError Traceback (most recent call last)
Cell In[23], line 1
----> 1 scraped_ds.push_to_hub("my_data_repo")
File /opt/conda/lib/python3.10/site-packages/datasets/dataset_dict.py:899, in DatasetDict.push_to_hub(self, repo_id, private, token, branch, shard_size, embed_external_files)
897 logger.warning(f"Pushing split {split} to the Hub.")
898 # The split=key needs to be removed before merging
--> 899 repo_id, split, uploaded_size, dataset_nbytes = self[split]._push_parquet_shards_to_hub(
900 repo_id,
901 split=split,
902 private=private,
903 token=token,
904 branch=branch,
905 shard_size=shard_size,
906 embed_external_files=embed_external_files,
907 )
908 total_uploaded_size += uploaded_size
909 total_dataset_nbytes += dataset_nbytes
File /opt/conda/lib/python3.10/site-packages/datasets/arrow_dataset.py:3474, in Dataset._push_parquet_shards_to_hub(self, repo_id, split, private, token, branch, shard_size, embed_external_files)
3472 shard.to_parquet(buffer)
3473 uploaded_size += buffer.tell()
-> 3474 _retry(
3475 api.upload_file,
3476 func_kwargs=dict(
3477 path_or_fileobj=buffer.getvalue(),
3478 path_in_repo=path_in_repo(index),
3479 repo_id=repo_id,
3480 token=token,
3481 repo_type="dataset",
3482 revision=branch,
3483 identical_ok=True,
3484 ),
3485 exceptions=HTTPError,
3486 status_codes=[504],
3487 base_wait_time=2.0,
3488 max_retries=5,
3489 max_wait_time=20.0,
3490 )
3491 return repo_id, split, uploaded_size, dataset_nbytes
File /opt/conda/lib/python3.10/site-packages/datasets/utils/file_utils.py:330, in _retry(func, func_args, func_kwargs, exceptions, status_codes, max_retries, base_wait_time, max_wait_time)
328 while True:
329 try:
--> 330 return func(*func_args, **func_kwargs)
331 except exceptions as err:
332 if retry >= max_retries or (status_codes and err.response.status_code not in status_codes):
File /opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:118, in validate_hf_hub_args.<locals>._inner_fn(*args, **kwargs)
115 if check_use_auth_token:
116 kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_token, kwargs=kwargs)
--> 118 return fn(*args, **kwargs)
File /opt/conda/lib/python3.10/site-packages/huggingface_hub/hf_api.py:826, in future_compatible.<locals>._inner(self, *args, **kwargs)
823 return self.run_as_future(fn, self, *args, **kwargs)
825 # Otherwise, call the function normally
--> 826 return fn(self, *args, **kwargs)
TypeError: HfApi.upload_file() got an unexpected keyword argument 'identical_ok'
解决方法
更新版本
!pip install -U datasets huggingface-hub
重启kernal:点击 factory restrat