Tensorflow 入门回顾——数据处理基本流程

Boston Housing，波士顿房价趋势数据集，用于回归模型训练与测试。

CIFAR10/100，真实图片数据集，用于图片分类任务。

MNIST/Fashion_MNIST，手写数字图片数据集，用于图片分类任务。IMDB，情感分类任务数据集，用于文本分类任务。

1、数据集加载：

from tensorflow.keras import datasets


(train_x,train_y),(test_x,test_y) = datasets.mnist.load_data()


print(train_x.shape,train_y.shape,test_x.shape,test_y.shape)
print(train_x[0:1])

(60000, 28, 28) (60000,) (10000, 28, 28) (10000,)
[[[  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   3  18  18  18 126
   136 175  26 166 255 247 127   0   0   0   0]
  [  0   0   0   0   0   0   0   0  30  36  94 154 170 253 253 253 253
   253 225 172 253 242 195  64   0   0   0   0]
  [  0   0   0   0   0   0   0  49 238 253 253 253 253 253 253 253 253
   251  93  82  82  56  39   0   0   0   0   0]
  [  0   0   0   0   0   0   0  18 219 253 253 253 253 253 198 182 247
   241   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0  80 156 107 253 253 205  11   0  43
   154   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0  14   1 154 253  90   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0 139 253 190   2   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0  11 190 253  70   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0  35 241 225 160 108
     1   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0  81 240 253 253
   119  25   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0  45 186 253
   253 150  27   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  16  93
   252 253 187   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
   249 253 249  64   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0  46 130 183
   253 253 207   2   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0  39 148 229 253 253
   253 250 182   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0  24 114 221 253 253 253 253
   201  78   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0  23  66 213 253 253 253 253 198  81
     2   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0  18 171 219 253 253 253 253 195  80   9   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0  55 172 226 253 253 253 253 244 133  11   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0 136 253 253 253 212 135 132  16   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]
  [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0
     0   0   0   0   0   0   0   0   0   0   0]]]

2、数据集转换成dataset

train_data = tf.data.Dataset.from_tensor_slices((train_x,train_y))
print(train_data)

<TensorSliceDataset shapes: ((28, 28), ()), types: (tf.uint8, tf.uint8)>

3、数据随机打散

train_data = train_data.shuffle(10000)
print(train_data)

<ShuffleDataset shapes: ((28, 28), ()), types: (tf.uint8, tf.uint8)>

4、数据样本批训练，即将训练集分成多个batch

train_data = train_data.batch(128)
print(train_data)

<BatchDataset shapes: ((None, 28, 28), (None,)), types: (tf.uint8, tf.uint8)>

5、预处理

def preprocess(x, y): # 自定义的预处理函数
# 调用此函数时会自动传入x,y 对象，shape 为[b, 28, 28], [b]
# 标准化到0~1
    x = tf.cast(x, dtype=tf.float32) / 255.
    x = tf.reshape(x, [-1, 28*28]) # 打平
    y = tf.cast(y, dtype=tf.int32) # 转成整型张量
    y = tf.one_hot(y, depth=10) # one-hot 编码
    # 返回的x,y 将替换传入的x,y 参数，从而实现数据的预处理功能
    return x,y

train_data = train_data.map(preprocess)

<MapDataset shapes: ((None, 784), (None, 10)), types: (tf.float32, tf.float32)>

6、迭代循环训练

def train_epoch(epoch):


    for step,(x,y) in enumerate(train_dataset):


        with tf.GradientTape() as tape: #构建梯度记录环境
            x = tf.reshape(x,(-1,28*28))  #打平操作
            out = model(x)
            loss = tf.reduce_sum(tf.square(out-y))/tf.cast(x.shape[0],tf.float32)
            #自动计算梯度
        grads = tape.gradient(loss,model.trainable_variables)


            #更新网络参数 w1= w-lr*grad
        optimizer.apply_gradients(zip(grads,model.trainable_variables))


        if step % 100 ==0:
            print(epoch,step,'loss:',loss.numpy())

7、启动训练

def train():
    for epoch in range(30):
        train_epoch(epoch)


if __name__ == '__main__':
    train()

听说关注公众号的都是大牛：