问题描述:
用
https://huggingface.co/datasets/HuggingFaceTB/smollm-corpus
这里面的代码下载会报错
正确下载代码:
import os
import json
import boto3
import gzip
from botocore.client import Config #debug
from botocore import UNSIGNED #debug
from datasets import load_dataset
os.environ["NO_PROXY"] = "s3.amazonaws.com"
num_proc = 16
s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED)) #debug
bucket_name = "softwareheritage"
save_dir = "/Smollm-corpus/python-edu"