数据洗牌

在实验过程中，可能出现海量数据但是过拟合的现象，这种情况可能是由于train数据和val/test的数据差别较大引起的。在这种情况下可以进行数据洗牌，即通过重新调整数据，将数据重新打乱按照之前的分配比例重新分割数据，形成新的train/val/test这种方式不会损坏数据原有的结构，可能消除过拟合的现象。

详细代码如下：

#coding=utf-8
import numpy as np
import os
import glob
import time
from random import shuffle
import os.path
import shutil

path = ''
path_new = ''


#创建新的文件夹存放
def new_dir(_new_dir):
    """new paht save image"""
    if not os.path.exists(_new_dir):
        os.makedirs(_new_dir)
        #print _new_dir+" has been created"


def moveFileto(oldDir, newDir, falg):
    #复制文件并重命名
    shutil.copy(oldDir, newDir+'/'+falg)


def save_npy(path_load):

    data_files = glob.glob(path_load + '*/' + '/*.npy')
    list = []
    print (path_load)
    for j, i in enumerate(data_files):
        list.append(i)
    shuffle(list)
    for j, i in enumerate(list):
        paths = i.split('/')[7:]
        #print (paths)
        if j < 50:
            new_path = path_new + '/' + paths[0] + '/val'
            falg = paths[0]+'_'+str(j).zfill(5)+'.npy'
        elif j >= 50 and  j < 100:
            new_path = path_new + '/' + paths[0] + '/test'
            falg = paths[0] + '_' + str(j - 49).zfill(5) + '.npy'
        else:
            new_path = path_new + '/' + paths[0] + '/train'
            falg = paths[0]+'_'+str(j-99).zfill(5)+'.npy'
        new_dir(new_path)
        moveFileto(i, new_path, falg)


def load_path():

    with open('./label_sorted.txt') as myfile:
        data_dir = myfile.read().splitlines()

    for j in range(len(data_dir)):
        #print data_dir[j]
        path_load = path + '/'+data_dir[j] +'/'
        #print path_load
        save_npy(path_load)


load_path()