Tensorflow（二十七） —— 过拟合与欠拟合

1. 过拟合检测

# ************************* 1、how to detect overfitting
# 1.1 splitting
(x,y),(x_test,y_test) = keras.datasets.mnist.load_data()
x_train,x_val = tf.split(x,num_or_size_splits=[50000,10000])
y_train,y_val = tf.split(y,num_or_size_splits = [50000,10000])

def preprocess(x,y):
    # x是一张照片
    x = tf.reshape(tf.cast(x,dtype=tf.float32)/255.,[28*28])
    y = tf.one_hot(tf.cast(y,dtype=tf.int32),depth=10)
    return x,y
db_train = tf.data.Dataset.from_tensor_slices((x_train,y_train))
db_val = tf.data.Dataset.from_tensor_slices((x_val,y_val))

db_train = db_train.map(preprocess).shuffle(10000).batch(128)
db_val = db_val.map(preprocess).shuffle(10000).batch(128)
"""
案例
"""
network = keras.Sequential([
                            keras.layers.Dense(28*28,activation = tf.nn.relu),
                            keras.layers.Dense(256,activation = tf.nn.relu),
                            keras.layers.Dense(10)
])
network.build(input_shape = [None,28*28])

network.compile(optimizer = keras.optimizers.Adam(lr = 1e-3),\
               loss = tf.losses.CategoricalCrossentropy(from_logits = True),\
               metrics = ["accuracy"])
network.fit(db_train,epochs = 10,validation_data = db_val,validation_freq = 2)

network.evaluate(db_val)

2. k折交叉验证

# 1.3 k-fold cross-validation
"""
交叉验证
1、merge train/val sets
2、random sample 1/k as val set
"""
# 自行实现
for epoch in range(500):
    index = tf.range(60000)
    index = tf.random.shuffle(index)
    x_train,y_train = tf.gather(x,index[:5000]),tf.gather(y,index[:5000])
    x_val,y_val = tf.gather(x,index[-1000:]),tf,gather(y,index[-1000:])
    
# 简捷
network.fit(db_train,\
            validation_split = 0.1,\
           validation_freq = 2,\
           epochs = 10)

3. 过拟合的解决方案

3.1 正则化

# 2.1 regularization
"""
weight decay
"""
# 用法一
L2_model = keras.Sequential([
                            keras.layers.Dense(256,kernel_regularizer = keras.regularizers.l2(0.001),\
                                               bias_regularizer = keras.regularizers.l2(0.001),# 自由设定 惩罚度不同
                                              activation = tf.nn.relu),
                            keras.layers.Dense(128,kernel_regularizer = keras.regularizers.l2(0.001),\
                                              bias_regularizer = keras.regularizers.l2(0.001),
                                              activation = tf.nn.relu),
                            keras.layers.Dense(64,kernel_regularizer = keras.regularizers.l2(0.001),\
                                              bias_regularizer = keras.regularizers.l2(0.001),
                                              activation = tf.nn.relu),
                            keras.layers.Dense(10,activation = tf.nn.sigmoid)
])
# 用法二
for step,(x,y) in enumerate(db_train):
    with tf.GradientTape() as tape:
        loss = tf.losses.categorical_crossentropy(y_onehot,out,from_logits=True)
        loss_regularizer = []
        for p in network.trainable_variables:
            loss_regularizer.append(tf.nn.l2_loss(p))
        loss_regularizer = tf.reduce_sum(tf.stack(loss_regularizer))
        loss = loss + 0.0001*loss_regularizer
    
    grads = tape.gradient(loss,network.trainable_variables)
    optimizer.apply_gradients(zip(grads,network.trainable_variables))

3.2 动量与学习率

# 2.2 动量与学习率
"""
momentum 动量
learning rate decay
"""
# momentum
optimizer = keras.optimizers.SGD(learning_rate = 0.001,momentum=0.9) # 动量因子一般设定为0.9
optimizer = keras.optimizers.RMSprop(learning_rate=0.001,momentum=0.9) # 此处也是动量因子，指数加权平均系数已默认设置为0.9
optimizer = keras.optimizers.Adam(learning_rate=0.001,beta_1 = 0.9,beta_2 = 0.999) # beta_1为动量因子，beta_2为指数加权平均系数

# learning rate decay
optimizer = keras.optimizers.SGD(learning_rate=0.2)
for epoch in range(100):
    optimizer.learning_rate = 0.2 * (100-epoch)/100

3.3 early stopping / Dropout / stochastic gradient descent

# Dropout
network = keras.models.Sequential([
                                    keras.layers.Dense(256,activation = tf.nn.relu),\
                                    keras.layers.Dropout(0.5),\
                                    keras.layers.Dense(128,activation = tf.nn.relu),\
                                    keras.layers.Dropout(0.5),\
                                    keras.layers.Dense(64,activation = tf.nn.relu),\
                                    keras.layers.Dense(32,activation = tf.nn.relu),\
                                    keras.layers.Dense(10)
])

# 针对于训练集
for step,(x,y) in enumerate(db):
    out = network(x,training = True)
    
# 针对于验证集
out = network(x,training = False)

# 针对于测试集
out = network(x,training = False)

本文为参考龙龙老师的“深度学习与TensorFlow 2入门实战“课程书写的学习笔记

by CyrusMay 2022 04 18