处理imagenet2012数据集

下载的imagenet2012数据集是
ILSVRC2012_img_train.tar 和 ILSVRC2012_img_val.tar这样的两个数据
首先解压文件

mkdir train
mkdir val
# tar xvf ILSVRC2012_img_train.tar -C ./train
tar xvf '/root/data/public/ImageNet 2012数据集/ILSVRC2012_img_train.tar' -C ./train
tar xvf '/root/data/public/ImageNet 2012数据集/ILSVRC2012_img_val.tar' -C ./val
# tar xvf ILSVRC2012_img_val.tar -C ./val

对于train的压缩包，解压之后其实还是1000个tar压缩包（对应1000个类别），需要再次解压

import tarfile
import os
def un_tar(file_name):
    tar = tarfile.open(file_name)
    file_name = file_name.split('.')[0]
    """解压tar"""
    print(file_name)
    names = tar.getnames()
    temp_file_path = ''
    if os.path.isdir(file_name):
        print('文件已存在')
        temp_file_path = os.path.isdir(file_name)
    else:
        temp_file_path = os.mkdir(file_name)
        print('创建一个新的文件名')
    #因为解压后是很多文件，预先建立同名目录
    for name in names:
        tar.extract(name, file_name)
    tar.close()
    return temp_file_path
filePath = 'train'
for tar in os.listdir(filePath):
    un_tar(filePath + '/' + tar)
    os.remove(filePath + '/' + tar)

整理测试集（需要ILSVRC2012_devkit_t12.tar.gz）

from scipy import io
import os
import shutil

def move_valimg(val_dir='./val', devkit_dir='./ILSVRC2012_devkit_t12'):
    """
    move valimg to correspongding folders.
    val_id(start from 1) -> ILSVRC_ID(start from 1) -> WIND
    organize like:
    /val
       /n01440764
           images
       /n01443537
           images
        .....
    """
    # load synset, val ground truth and val images list
    synset = io.loadmat(os.path.join(devkit_dir, 'data', 'meta.mat'))
    
    ground_truth = open(os.path.join(devkit_dir, 'data', 'ILSVRC2012_validation_ground_truth.txt'))
    lines = ground_truth.readlines()
    labels = [int(line[:-1]) for line in lines]
    
    root, _, filenames = next(os.walk(val_dir))
    for filename in filenames:
        # val image name -> ILSVRC ID -> WIND
        val_id = int(filename.split('.')[0].split('_')[-1])
        ILSVRC_ID = labels[val_id-1]
        WIND = synset['synsets'][ILSVRC_ID-1][0][1][0]
        print("val_id:%d, ILSVRC_ID:%d, WIND:%s" % (val_id, ILSVRC_ID, WIND))

        # move val images
        output_dir = os.path.join(root, WIND)
        if os.path.isdir(output_dir):
            pass
        else:
            os.mkdir(output_dir)
        shutil.move(os.path.join(root, filename), os.path.join(output_dir, filename))

if __name__ == '__main__':
    move_valimg()