tflearn 应用cnn完成cifar10数据集训练和预测

实现的内容：
参照原tflearn example，进行改动：
1. 对cifar10数据集完成了预处理，将原5个训练集进行了划分，划分为训练集和验证集；
2. 使用tflearn搭建CNN网络，使用了三层卷积+最大池化
3. 对搭建的CNN网络，在原测试集上进行了评估
4. 对任意一张图片进行预测；
5. 将模型转移到阿里云PCA，储存在云端；

预处理&训练代码：

# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import

import tensorflow as tf

from six.moves import urllib
import tarfile

from tensorflow.python.lib.io import file_io
import os
import sys
import numpy as np
import pickle
import argparse
import scipy

from sklearn.preprocessing import LabelBinarizer


FLAGS = None


# 本地读取可以直接使用with open():
# ```
# def load_cfar10_batch(cifar10_dataset_folder_path, batch_id):
#     """
#     Load a batch of the dataset
#     """
#     with open(cifar10_dataset_folder_path + '/data_batch_' + str(batch_id), mode='rb') as file:
#         batch = pickle.load(file, encoding='latin1')
# 
#     features = batch['data'].reshape(-1, 3, 32, 32)).transpose(0, 2, 3, 1)
#     labels = batch['labels']
# 
#     return features, labels
# ```
# 
# 注意：features = batch['data'].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1)
# 原数据是1024（32*32）个红，1024个绿，1024个蓝色，依次循环的数据，所以先resha成3，32，32，再转置成32,32,3

def load_batch(fpath):
    object = file_io.read_file_to_string(fpath)
    #origin_bytes = bytes(object, encoding='latin1')
    # with open(fpath, 'rb') as file:
    # batch = pickle.load(file, encoding='latin1')
    if sys.version_info > (3, 0):
        # Python3
        batch = pickle.loads(object, encoding='latin1')
    else:
        # Python2
        batch = pickle.loads(object)
    features = batch["data"].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1)
    labels = batch["labels"]
    return features, labels


def normalize(x):
    """
    Normalize a list of sample image data in the range of 0 to 1
    : x: List of image data.  The image shape is (32, 32, 3)
    : return: Numpy array of normalize data
    """
    x = (x - np.min(x))/(np.max(x) - np.min(x))
    return x

# from sklearn.preprocessing import LabelBinarizer
enc = LabelBinarizer()
enc.fit(range(10))

def one_hot_encode(x):
    """
    One hot encode a list of sample labels. Return a one-hot encoded vector for each label.
    : x: List of sample Labels
    : return: Numpy array of one-hot encoded labels
    """
    return enc.transform(x)


def load_data(dirname):
    X_train = []
    Y_train = []

    for i in range(1, 6):
        fpath = os.path.join(dirname, 'data_batch_' + str(i))
        data, labels = load_batch(fpath)
        if i == 1:
            X_train = data
            Y_train = labels
        else:
            X_train = np.concatenate([X_train, data], axis=0)
            Y_train = np.concatenate([Y_train, labels], axis=0)

    fpath = os.path.join(dirname, 'test_batch')
    X_test, Y_test = load_batch(fpath)

    X_valid = []
    Y_valid = []
    validation_count = int(len(X_train) * 0.1)
    X_valid.extend(X_train[-validation_count:])
    Y_valid.extend(Y_train[-validation_count:])


    X_train = normalize(X_train[:-validation_count])
    Y_train = one_hot_encode(Y_train[:-validation_count])
    X_valid = normalize(X_valid)
    Y_valid = one_hot_encode(Y_valid)
    X_test = normalize(X_test)
    Y_test = one_hot_encode(Y_test)



    return (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test)


#reporthook from stackoverflow #13881092
def reporthook(blocknum, blocksize, totalsize):
    readsofar = blocknum * blocksize
    if totalsize > 0:
        percent = readsofar * 1e2 / totalsize
        s = "\r%5.1f%% %*d / %d" % (
            percent, len(str(totalsize)), readsofar, totalsize)
        sys.stderr.write(s)
        if readsofar >= totalsize: # near the end
            sys.stderr.write("\n")
    else: # total size is unknown
        sys.stderr.write("read %d\n" % (readsofar,))


import tflearn
from tflearn.data_utils import shuffle, to_categorical
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation


def main(_):
    dirname = os.path.join(FLAGS.buckets, "")
    (X, Y), (X_valid, Y_valid), (X_test, Y_test) = load_data(dirname)
    print("load data done")

    X, Y = shuffle(X, Y)

    # Real-time data preprocessing
    img_prep = ImagePreprocessing()
    img_prep.add_featurewise_zero_center()
    img_prep.add_featurewise_stdnorm()

    # Real-time data augmentation
    img_aug = ImageAugmentation()
    img_aug.add_random_flip_leftright()
    img_aug.add_random_rotation(max_angle=25.)

    # Convolutional network building
    network = input_data(shape=[None, 32, 32, 3],
                         data_preprocessing=img_prep,
                         data_augmentation=img_aug)
    network = conv_2d(network, 32, 3, activation='relu')
    network = max_pool_2d(network, 2)
    network = conv_2d(network, 64, 3, activation='relu')
    network = max_pool_2d(network, 3)
    network = conv_2d(network, 64, 3, activation='relu')
    network = max_pool_2d(network, 2)
    network = fully_connected(network, 512, activation='relu')
    network = dropout(network, 0.5)
    network = fully_connected(network, 10, activation='softmax')
    network = regression(network, optimizer='adam',
                         loss='categorical_crossentropy',
                         learning_rate=0.001)

    # Train using classifier
    model = tflearn.DNN(network, tensorboard_verbose=0)
    model.fit(X, Y, n_epoch=50, shuffle=True, validation_set=(X_valid, Y_valid),
              show_metric=True, batch_size=96, run_id='cifar10_cnn')
    model_path = os.path.join(FLAGS.checkpointDir, "model.tfl")
    print(model_path)
    model.save(model_path)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    #获得buckets路径
    parser.add_argument('--buckets', type=str, default='',
                        help='input data path')
    #获得checkpoint路径
    parser.add_argument('--checkpointDir', type=str, default='',
                        help='output model path')
    FLAGS, _ = parser.parse_known_args()
    tf.app.run(main=main)

在测试集上restore，并evaluate：

# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import

import tensorflow as tf

from six.moves import urllib
import tarfile

from tensorflow.python.lib.io import file_io
import os
import sys
import numpy as np
import pickle
import argparse
import scipy

from sklearn.preprocessing import LabelBinarizer


FLAGS = None

def load_batch(fpath):
    object = file_io.read_file_to_string(fpath)
    #origin_bytes = bytes(object, encoding='latin1')
    # with open(fpath, 'rb') as file:
    # batch = pickle.load(file, encoding='latin1')
    if sys.version_info > (3, 0):
        # Python3
        batch = pickle.loads(object, encoding='latin1')
    else:
        # Python2
        batch = pickle.loads(object)
    features = batch["data"].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1)
    labels = batch["labels"]
    return features, labels


def normalize(x):
    """
    Normalize a list of sample image data in the range of 0 to 1
    : x: List of image data.  The image shape is (32, 32, 3)
    : return: Numpy array of normalize data
    """
    x = (x - np.min(x))/(np.max(x) - np.min(x))
    return x

# from sklearn.preprocessing import LabelBinarizer
enc = LabelBinarizer()
enc.fit(range(10))

def one_hot_encode(x):
    """
    One hot encode a list of sample labels. Return a one-hot encoded vector for each label.
    : x: List of sample Labels
    : return: Numpy array of one-hot encoded labels
    """
    return enc.transform(x)


def load_data(dirname):   
    fpath = os.path.join(dirname, 'test_batch')
    X_test, Y_test = load_batch(fpath)

    X_test = normalize(X_test)
    Y_test = one_hot_encode(Y_test)

    return (X_test, Y_test)


#reporthook from stackoverflow #13881092
def reporthook(blocknum, blocksize, totalsize):
    readsofar = blocknum * blocksize
    if totalsize > 0:
        percent = readsofar * 1e2 / totalsize
        s = "\r%5.1f%% %*d / %d" % (
            percent, len(str(totalsize)), readsofar, totalsize)
        sys.stderr.write(s)
        if readsofar >= totalsize: # near the end
            sys.stderr.write("\n")
    else: # total size is unknown
        sys.stderr.write("read %d\n" % (readsofar,))


import tflearn
from tflearn.data_utils import shuffle, to_categorical
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation

def main(_):
    dirname = os.path.join(FLAGS.buckets, "")
    (X_test, Y_test) = load_data(dirname)
    print("load data done")

    # Real-time data preprocessing
    img_prep = ImagePreprocessing()
    img_prep.add_featurewise_zero_center()
    img_prep.add_featurewise_stdnorm()

    # Real-time data augmentation
    img_aug = ImageAugmentation()
    img_aug.add_random_flip_leftright()
    img_aug.add_random_rotation(max_angle=25.)

    # Convolutional network building
    network = input_data(shape=[None, 32, 32, 3],
                         data_preprocessing=img_prep,
                         data_augmentation=img_aug)
    network = conv_2d(network, 32, 3, activation='relu')
    network = max_pool_2d(network, 2)
    network = conv_2d(network, 64, 3, activation='relu')
    network = max_pool_2d(network, 3)
    network = conv_2d(network, 64, 3, activation='relu')
    network = max_pool_2d(network, 2)
    network = fully_connected(network, 512, activation='relu')
    network = dropout(network, 0.5)
    network = fully_connected(network, 10, activation='softmax')
    network = regression(network, optimizer='adam',
                         loss='categorical_crossentropy',
                         learning_rate=0.001)

    # Test using model
    model = tflearn.DNN(network, tensorboard_verbose=0)
    model_path = os.path.join(FLAGS.checkpointDir, "model.tfl")
    print(model_path)
    model.load(model_path)

    result = model.evaluate(X_test, Y_test)
    print(result)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    #获得buckets路径
    parser.add_argument('--buckets', type=str, default='',
                        help='input data path')
    #获得checkpoint路径
    parser.add_argument('--checkpointDir', type=str, default='',
                        help='output model path')
    FLAGS, _ = parser.parse_known_args()
    tf.app.run(main=main)

导入任一张图片，进行预测：

# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import

import tensorflow as tf

from six.moves import urllib
import tarfile

import tflearn
from tflearn.data_utils import shuffle, to_categorical
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation

from tensorflow.python.lib.io import file_io
import os
import sys
import numpy as np
import pickle
import argparse
import scipy
FLAGS = None

    ```
    #与test中代码相同，省略……
    ```
def main(_):
    ```
    #与test中代码相同，省略……
    ```
    # Train using classifier
    model = tflearn.DNN(network, tensorboard_verbose=0)
    # model.fit(X, Y, n_epoch=100, shuffle=True, validation_set=(X_test, Y_test),
    #           show_metric=True, batch_size=96, run_id='cifar10_cnn')
    model_path = os.path.join(FLAGS.checkpointDir, "model.tfl")
    print(model_path)
    model.load(model_path)



    predict_pic = os.path.join(FLAGS.buckets, "bird_bullocks_oriole.jpg")
    img_obj = file_io.read_file_to_string(predict_pic)
    file_io.write_string_to_file("bird_bullocks_oriole.jpg", img_obj)

    img = scipy.ndimage.imread("bird_bullocks_oriole.jpg", mode="RGB")

    # Scale it to 32x32
    img = scipy.misc.imresize(img, (32, 32), interp="bicubic").astype(np.float32, casting='unsafe')

    # Predict
    prediction = model.predict([img])
    print (prediction[0])
    print (prediction[0])
    #print (prediction[0].index(max(prediction[0])))
    num=['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']
    print ("This is a %s"%(num[prediction[0].tolist().index(max(prediction[0]))]))


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    #获得buckets路径
    parser.add_argument('--buckets', type=str, default='',
                        help='input data path')
    #获得checkpoint路径
    parser.add_argument('--checkpointDir', type=str, default='',
                        help='output model path')
    FLAGS, _ = parser.parse_known_args()
    tf.app.run(main=main)