TensorFlow 2.0 笔记（六）—— 过拟合与欠拟合

最新推荐文章于 2022-06-06 20:43:29 发布

资深糖分大叔

最新推荐文章于 2022-06-06 20:43:29 发布

阅读量1.2k

点赞数 2

分类专栏：机器学习 TensorFlow2.0

本文链接：https://blog.csdn.net/hetengjiao523/article/details/101511988

版权

机器学习同时被 2 个专栏收录

24 篇文章 6 订阅

订阅专栏

TensorFlow2.0

9 篇文章 1 订阅

订阅专栏

Mismatch: ground-truth VS estimated

model capacity
$y=\beta_0+\beta_1x+\beta_2x^2+\beta_3x^3+...+\beta_nx^n+\epsilon.$
under-fitting: Estimated < ground-truth
- train acc. is bad
- test acc. is bad as well
over-fitting: Estimated > ground-truth
- train loss and acc. is much better
- test acc. is worse
- => Generalization Performance

Detect over-fitting
split dataset: Train Set, Val Set, Test Set

dataset, metadata = tfds.load('fashion_mnist', as_supervised=True, with_info=True)
train_dataset, test_data = dataset['train'], dataset['test']

num_val_examples = 10000
num_train_examples = metadata.splits['train'].num_examples - num_val_examples
num_test_examples = metadata.splits['test'].num_examples

# 将 train数据集划分为 train和 validate两部分
train_data = train_dataset.take(num_train_examples)
val_data = train_dataset.skip(num_train_examples).take(num_val_examples)

(x, y), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()

idx = tf.range(60000)
idx = tf.random.shuffle(idx)
x_train, y_train = tf.gather(x, idx[:50000]), tf.gather(y, idx[:50000])
x_val, y_val = tf.gather(x, idx[-10000:]), tf.gather(y, idx[-10000:])

train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
val_data = tf.data.Dataset.from_tensor_slices((x_val, y_val))
test_data = tf.data.Dataset.from_tensor_slices((x_test, y_test))

train test trade-off
K-fold cross-validation

merge train/val sets
randomly sample 1/k as val set

model.fit(train_data, epochs=10, validation_split=0.1, validation_freq=1)

Reduce Overfitting

More data
Constraint model complexity
- shallow
- regularization
Dropout
Data argumentation
Early Stopping

import os
import tensorflow as tf

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

(x, y), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()


idx = tf.range(60000)
idx = tf.random.shuffle(idx)
x_train, y_train = tf.gather(x, idx[:50000]), tf.gather(y, idx[:50000])
x_val, y_val = tf.gather(x, idx[-10000:]), tf.gather(y, idx[-10000:])

train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
val_data = tf.data.Dataset.from_tensor_slices((x_val, y_val))
test_data = tf.data.Dataset.from_tensor_slices((x_test, y_test))
train_val_data = zip(tf.convert_to_tensor(x),tf.convert_to_tensor(y))


def normalize(images, labels):
    images = tf.cast(images, tf.float32)
    images /= 255.
    images = tf.reshape(images, [28*28])
    labels = tf.one_hot(labels, depth=10)
    return images, labels


BATCH_SIZE = 100
train_data = train_data.map(normalize).shuffle(50000).batch(BATCH_SIZE)\
    .prefetch(tf.data.experimental.AUTOTUNE)
val_data = val_data.map(normalize).shuffle(10000).batch(BATCH_SIZE)\
    .prefetch(tf.data.experimental.AUTOTUNE)
test_data = test_data.map(normalize).batch(BATCH_SIZE)


class MyDense(tf.keras.layers.Layer):

    def __init__(self, inp_dim, outp_dim):
        super(MyDense, self).__init__()

        self.kernel = self.add_variable('w', [inp_dim, outp_dim])
        self.bias = self.add_variable('b', [outp_dim])

    def call(self, inputs, training=None):
        out = inputs @ self.kernel + self.bias

        return out


class MyModel(tf.keras.Model):

    def __init__(self):
        super(MyModel, self).__init__()
        self.fc1 = MyDense(28*28, 256)
        self.fc2 = MyDense(256, 128)
        self.fc3 = MyDense(128, 64)
        self.fc4 = MyDense(64, 32)
        self.fc5 = MyDense(32, 10)

    def call(self, inputs, training=None, mask=None):
        x = self.fc1(inputs)
        x = tf.nn.relu(x)
        x = self.fc2(x)
        x = tf.nn.relu(x)
        x = self.fc3(x)
        x = tf.nn.relu(x)
        x = self.fc4(x)
        x = tf.nn.relu(x)
        x = self.fc5(x)

        return x


model = MyModel()
model.build(input_shape=[None, [28*28]])
model.summary()
model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
              loss=tf.losses.CategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])
#model.fit(train_data, epochs=5, validation_data=val_data, validation_freq=1)
model.fit(train_val_x, train_val_y, epochs=5, validation_split=0.1, validation_freq=1)

model.evaluate(test_data)

Regularization
$J(\theta)=-\frac{1}{m}\sum_{i=1}^m[y_iln\hat{y_i}+(1-y_i)ln(1-\hat{y_i})]+\lambda\sum_{i=1}^n|\theta_i|$

L1-regularization
$J(\theta)=-\frac{1}{m}\sum_{i=1}^m[y_iln\hat{y_i}+(1-y_i)ln(1-\hat{y_i})]+\lambda\sum_{i=1}^n|\theta_i|$
L2-regularization
$J(W;X,y)+\frac{1}{2}\lambda\cdot||W||^2$

l2_model = tf.keras.models.Sequential([
    tf.keras.layers.Dense(16, kernel_regularizer=tf.keras.regularizers.l2(0.001), 
                          activation=tf.nn.relu,
                          input_shape=([None, 28*28])),
    tf.keras.layers.Dense(16, kernel_regularizer=tf.keras.regularizers.l2(0.001), 
                          activation=tf.nn.relu),
    tf.keras.layers.Dense(1, activation=tf.nn.sigmoid)
])

for step, (x,y) in enumerate(train_data):
    with tf.GradientTape() as tape:
        loss = tf.reduce_mean(tf.losses.categorical_crossentropy(y_onehot, out, from_logits=true))
        loss_regularization = []
        for p in model.trainable_variables:
            loss_regularization.append(tf.nn.l2_loss(p))
        loss_regularization = tf.reduce_sum(tf.stack(loss_regularization))

        loss = loss + 0.0001 * loss_regularization

    grads = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(grads, model.trainable_variables))

momentum
$w^{k+1}=w^k-\alpha\nabla f(w^k)$
$z^{k+1}=\beta z^k+\nabla f(w^k)$
$w^{k+1}=w^k-\alpha z^{k+1}$

optimizer = tf.keras.optimizers.SGD(learning_rate=0.02, momentum=0.9)
optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.02, momentum=0.9)
optimizer = tf.keras.optimizers.Adam(learning_rate=0.02, beta_1=0.9, beta_2=0.999)

learning rate decay

optimizer = tf.keras.optimizers.SGD(learning_rate=0.2)
for epoch in range(100):
    #get loss
    
    #change learning rate
    optimizer.learning_rate = 0.2 *(100 - epoch) / 100
    
    #update weights

Early Stopping

Validation set to select parameters
Monitor validation performance
Stop at the highest val perf

Dropout

Learning less to learn better
Each connection has $p = [0, 1]$ to lose

model = tf.keras.Sequential([
    tf.keras.layers.Dense(256, activation=tf.nn.relu),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(128, activation=tf.nn.relu),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(64, activation=tf.nn.relu),
    tf.keras.layers.Dense(32, activation=tf.nn.relu),
    tf.keras.layers.Dense(10)
])
for step, (x,y) in enumerate(train_data):
    with tf.GradientTape() as tape:
        x = tf.reshape(x, (-1, 28*28))
        # train
        out = model(x, training=True)
        # val out = model(x, training=False)
    # test
    out = model(x, training=False)

Stochastic Gradient Descent

Stochastic
- not random!
Deterministic
Not single usually
batch = 16, 32, 64, 128…

import os
import tensorflow as tf


os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

(x, y), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()


def normalize(images, labels):
    images = tf.cast(images, tf.float32)
    images /= 255.
    images = tf.reshape(images, [28 * 28])
    return images, labels


model = tf.keras.Sequential([
    tf.keras.layers.Dense(256, activation=tf.nn.relu),
    tf.keras.layers.Dense(128, activation=tf.nn.relu),
    tf.keras.layers.Dense(64, activation=tf.nn.relu),
    tf.keras.layers.Dense(32, activation=tf.nn.relu),
    tf.keras.layers.Dense(10)
])

model.build(input_shape=[None, 28*28])
model.summary()
optimizer = tf.keras.optimizers.Adam(lr=1e-3)

acc_meter = tf.keras.metrics.Accuracy()
loss_meter = tf.keras.metrics.Mean()
test_meter = tf.keras.metrics.Accuracy()

BATCH_SIZE = 128
test_data = tf.data.Dataset.from_tensor_slices((x_test, y_test))
test_data = test_data.map(normalize).batch(BATCH_SIZE)

for epoch in range(5):
    idx = tf.range(60000)
    idx = tf.random.shuffle(idx)
    x_train, y_train = tf.gather(x, idx[:50000]), tf.gather(y, idx[:50000])
    x_val, y_val = tf.gather(x, idx[-10000:]), tf.gather(y, idx[-10000:])

    # 分割数据集
    train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    train_data = train_data.map(normalize).shuffle(50000).batch(BATCH_SIZE)

    val_data = tf.data.Dataset.from_tensor_slices((x_val, y_val))
    val_data = val_data.map(normalize).shuffle(10000).batch(BATCH_SIZE)

    for step, (x_train, y_train) in enumerate(train_data):
        with tf.GradientTape() as tape:
            y_one = tf.one_hot(y_train, depth=10)
            logits = model(x_train)
            # loss_mse = tf.reduce_mean(tf.losses.MSE(y_, logits))
            loss_ce = tf.reduce_mean(tf.losses.categorical_crossentropy(y_one, logits, from_logits=True))
            loss_meter.update_state(loss_ce)

        grads = tape.gradient(loss_ce, model.trainable_variables)
        optimizer.apply_gradients(zip(grads, model.trainable_variables))
        
        # 验证
        if step % 10 == 0:
            for _, (x_val, y_val) in enumerate(val_data):
                logits = model(x_val)
                prob = tf.nn.softmax(logits, axis=1)
                pred = tf.argmax(prob, axis=1)
                acc_meter.update_state(y_val, pred)

            print(epoch, step, 'loss:', loss_meter.result().numpy(),
                  'Evaluate Acc:', acc_meter.result().numpy())
            loss_meter.reset_states()
            acc_meter.reset_states()

    # 测试
    for _, (x_test, y_test) in enumerate(test_data):
        logits = model(x_test)
        prob = tf.nn.softmax(logits, axis=1)
        pred = tf.argmax(prob, axis=1)
        test_meter.update_state(y_test, pred)
    print(epoch, 'Test Acc:', test_meter.result().numpy())
    test_meter.reset_states()