colab中数据集保存到drive与取出的方法

from google.colab import drive
drive.mount('/content/drive')

一、下载数据集

from datasets import load_dataset
max_length = 32  # Maximum length of the captions in tokens
coco_dataset_ratio = 50  # 50% of the COCO2014 dataset

# Load the COCO2014 dataset for training, validation, and testing splits
train_ds = load_dataset("HuggingFaceM4/COCO", split=f"train[:{coco_dataset_ratio}%]")
valid_ds = load_dataset("HuggingFaceM4/COCO", split=f"validation[:{coco_dataset_ratio}%]")
test_ds = load_dataset("HuggingFaceM4/COCO", split="test")

二、保存数据集

dataset_path = '/content/drive/My Drive/COCO_Dataset_all'
train_ds.save_to_disk(dataset_path + '/train')
valid_ds.save_to_disk(dataset_path + '/validation')
test_ds.save_to_disk(dataset_path + '/test')

三、取出数据集

dataset_path = '/content/drive/My Drive/COCO_Dataset'
from datasets import load_from_disk

train_ds = load_from_disk(dataset_path + '/train')
valid_ds = load_from_disk(dataset_path + '/validation')
test_ds = load_from_disk(dataset_path + '/test')

四、删除文件夹

import os
import shutil

folder_path = '/content/drive/My Drive/要删除的文件夹'

# 检查文件夹是否存在
if os.path.exists(folder_path):
    # 使用 shutil.rmtree 删除文件夹
    shutil.rmtree(folder_path)
    print(f"文件夹 {folder_path} 已被删除。")
else:
    print(f"文件夹 {folder_path} 不存在。")