第一次参加kaggle的经验（小白专属）

    数据分析不是我所擅长的，可以直接使用别人的读取代码，有eda的更好了，省劲，首先找一个好的，可以提交的代码跑一下提交，这是第一步。

接下来，看看评论，哪种模型结构更好，站在巨人的肩膀上更轻松一点，现在主流都是残差网络了，需要尝试的也不是很多，如果比赛刚开始的话，不要着急，先进行数据分析，把数据下载下来不要着急，慢慢看，过一段时间评论会给出不少好的方案，还是一句话，站在巨人的肩膀上会更加轻松，第一次不需要尝试太多可能，先找一种跑跑看。
如果还没人开始，那就可以尝深度残差网络，resnet densenet efficientnet这三种网络，一般来说，这三个是老大了。
下面附一份keras的头文件

from tensorflow import keras

from keras.models import Model, Input, load_model

from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import ModelCheckpoint
from keras.callbacks import EarlyStopping


from keras.preprocessing.image import ImageDataGenerator

from keras.layers import Dense
from keras.layers import Conv2D
from keras.layers import Flatten
from keras.layers import MaxPool2D
from keras.layers import Dropout
from keras.layers import BatchNormalization
from keras.layers import Input
from keras.layers import concatenate
from keras.layers import GlobalAveragePooling2D
from keras.layers import AveragePooling2D
from keras.layers import Activation
from keras.layers import GlobalMaxPooling2D
from keras.layers import Reshape
from keras.layers import multiply
from keras.layers import Permute
from keras.layers import Add
from keras.layers import Lambda
from keras import backend as K
from keras.activations import sigmoid

from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19
from keras.applications.inception_v3 import InceptionV3
from keras.applications.xception import Xception
from keras.applications.inception_resnet_v2 import InceptionResNetV2
from keras.applications.mobilenet import MobileNet
from keras.applications.nasnet import NASNetLarge
from keras.applications.nasnet import NASNetMobile
from keras.applications.mobilenet_v2 import MobileNetV2
from keras.applications import DenseNet121
from keras.applications import DenseNet169
from keras.applications import DenseNet201
from keras.applications.resnet import ResNet50
from keras.applications.resnet import ResNet101
from keras.applications.resnet import ResNet152
from keras.applications.resnet_v2 import ResNet50V2
from keras.applications.resnet_v2 import ResNet101V2
from keras.applications.resnet_v2 import ResNet152V2
# !pip install -q efficientnet

import efficientnet.tfkeras as efn
#使用
#model = efn.EfficientNetB0

basic_model= DenseNet121(weights="imagenet", include_top=False)

这是我整理的包含全部的常用的keras的头文件，直接建立一个cell复制过去就可以了，反正也不占很大空间，省心。
想换别的结构只需要把DenseNet121换了就可以了，这个有点基础就应该懂了。
再分享一个多输出的模型代码

def build_model():
    densenet = DenseNet121(weights="imagenet", include_top=False)

    input = Input(shape=(SIZE, SIZE, N_ch))
    x = Conv2D(1, (3, 3), padding='same')(input)
    x=concatenate([x,x,x])
    x = densenet(x)
    x = Dropout(0.5)(x)
    
    x = GlobalAveragePooling2D()(x)
#     x = BatchNormalization()(x)
#     x = Dropout(0.5)(x)
#     x = Dense(256, activation='relu')(x)
#     x = BatchNormalization()(x)
#     x = Dropout(0.5)(x)

    # multi output
    grapheme_root = Dense(types['grapheme_root'],
                          activation = 'softmax', name='root')(x)
    vowel_diacritic = Dense(types['vowel_diacritic'],
                            activation = 'softmax', name='vowel')(x)
    consonant_diacritic = Dense(types['consonant_diacritic'],
                                activation = 'softmax', name='consonant')(x)

    # model
    model = Model(input,
                  [grapheme_root, vowel_diacritic, consonant_diacritic])
    
    return model
model = build_model()

也可以用这种，其实差不多

def create_model():
    #pretrained_model = tf.keras.applications.MobileNetV2(input_shape=[*IMAGE_SIZE, 3], include_top=False)
    #pretrained_model = tf.keras.applications.Xception(input_shape=[*IMAGE_SIZE, 3], include_top=False)
    #pretrained_model = tf.keras.applications.VGG16(weights='imagenet', include_top=False ,input_shape=[*IMAGE_SIZE, 3])
    #pretrained_model = tf.keras.applications.ResNet50(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])
    #pretrained_model = tf.keras.applications.MobileNet(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])
    # EfficientNet can be loaded through efficientnet.tfkeras library (https://github.com/qubvel/efficientnet)
    pretrained_model = efn.EfficientNetB7(weights='imagenet', include_top=False)
    
    pretrained_model.trainable = True

    model = tf.keras.Sequential([
        pretrained_model,
        tf.keras.layers.GlobalAveragePooling2D(),        
        tf.keras.layers.Dense(num_classes, activation='softmax')
    ])

    model.compile(
        optimizer='adam',
        loss = 'categorical_crossentropy',
        metrics=['accuracy']
    )

    return model

#%%

model = create_model()
model.summary()

大概的使用现有的结构构造模型就是这样
在分享一个kaggle显示所有文件的，虽然这个建立的时候就会给，但是这么好的东西还是要发一下，偶尔可能忘记，因为在本地编辑会不写，直接上传可能就忘了

for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

接下里是一个数据增强，keras的数据增强还是很好的

class MultiOutputDataGenerator(keras.preprocessing.image.ImageDataGenerator):

    def flow(self,
             x,
             y=None,
             batch_size=32,
             shuffle=True,
             sample_weight=None,
             seed=None,
             save_to_dir=None,
             save_prefix='',
             save_format='png',
             subset=None):

        targets = None
        target_lengths = {}
        ordered_outputs = []
        for output, target in y.items():
            if targets is None:
                targets = target
            else:
                targets = np.concatenate((targets, target), axis=1)
            target_lengths[output] = target.shape[1]
            ordered_outputs.append(output)


        for flowx, flowy in super().flow(x, targets, batch_size=batch_size,
                                         shuffle=shuffle):
            target_dict = {}
            i = 0
            for output in ordered_outputs:
                target_length = target_lengths[output]
                target_dict[output] = flowy[:, i: i + target_length]
                i += target_length

            yield flowx, target_dict

使用方法

 datagen = MultiOutputDataGenerator(
            featurewise_center=False,  # set input mean to 0 over the dataset
            samplewise_center=False,  # set each sample mean to 0
            featurewise_std_normalization=False,  # divide inputs by std of the dataset
            samplewise_std_normalization=False,  # divide each input by its std
            zca_whitening=False,  # apply ZCA whitening
            rotation_range=8,  # randomly rotate images in the range (degrees, 0 to 180)
            zoom_range = 0.15, # Randomly zoom image 
            width_shift_range=0.15,  # randomly shift images horizontally (fraction of total width)
            height_shift_range=0.15,  # randomly shift images vertically (fraction of total height)
            horizontal_flip=False,  # randomly flip images
            vertical_flip=False)  # randomly flip images


        # This will just calculate parameters required to augment the given data. This won't perform any augmentations
        datagen.fit(x_train)

        # Fit the model
        history = model.fit_generator(datagen.flow(x_train, {'dense_23': y_train_root, 'dense_24': y_train_vowel, 'dense_25': y_train_consonant}, batch_size=batch_size),
                                  epochs = epochs, validation_data = (x_test, [y_test_root, y_test_vowel, y_test_consonant]), 
                                  steps_per_epoch=x_train.shape[0] // batch_size, 
                                  callbacks=[learning_rate_reduction_root, learning_rate_reduction_vowel, learning_rate_reduction_consonant]
                                     )

hiostory根据实际调整吧，有的时候数据增强不一定有用，挨个试选择有用的吧。

学习率是很重要的，sgd带动量和adam是最常用的，sgd上限高，adam的收敛速度快，自由选择
自动减少是个不错的选择，有耐心可以在多少轮之后在减少，这个需要放在callbacks里面

learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', 
                                            patience=3, 
                                            verbose=1,
                                            factor=0.2, 
                                            min_lr=0.000001)

sgd带动量，初始学习率也很关键，好象有的人说是3e-4来着，这个我没尝试过

sgd = keras.optimizers.SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True)

在tpu花卉分类上我也看到一种不错的学习率调整

LR_START = 0.00001
LR_MAX = 0.00005 
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = .8

def lrfn(epoch):
    if epoch < LR_RAMPUP_EPOCHS:
        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START
    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
        lr = LR_MAX
    else:
        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN
    return lr
    
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 #callbacks=[lr_callback]

介绍模型的写法

 enet = efn.EfficientNetB7(
        input_shape=(512, 512, 3),
        weights='imagenet',
        include_top=False
    )

    model = tf.keras.Sequential([
        enet,
        tf.keras.layers.GlobalAveragePooling2D(),
        tf.keras.layers.Dense(len(CLASSES), activation='softmax')
    ])

Sequential非常好用，直接叠加非常方便的
模型融合是最后的方法了，原理很简单，预测的结果乘以一个系数，另一个乘以1减这个系数再用np.argmax，例如这样

probabilities = best_alpha*model.predict(test_images_ds) + (1-best_alpha)*model2.predict(test_images_ds)
predictions = np.argmax(probabilities, axis=-1)

显示训练的图像也很重要

def display_training_curves(training, validation, title, subplot):
    if subplot%10==1: # set up the subplots on the first call
        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')
        plt.tight_layout()
    ax = plt.subplot(subplot)
    ax.set_facecolor('#F8F8F8')
    ax.plot(training)
    ax.plot(validation)
    ax.set_title('model '+ title)
    ax.set_ylabel(title)
    #ax.set_ylim(0.28,1.05)
    ax.set_xlabel('epoch')
    ax.legend(['train', 'valid.'])
#参数要看history里面有什么，好像是history.keys(),反正有key，记不大清楚了
display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)
display_training_curves(history.history['sparse_categorical_accuracy'], history.history['val_sparse_categorical_accuracy'], 'accuracy', 212)

裁剪也是一门艺术

def bbox(img):
    rows = np.any(img, axis=1)
    cols = np.any(img, axis=0)
    rmin, rmax = np.where(rows)[0][[0, -1]]
    cmin, cmax = np.where(cols)[0][[0, -1]]
    return rmin, rmax, cmin, cmax

def crop_resize(df, size=224, pad=16):
    resized = {}
    #crop a box around pixels large than the threshold 
    #some images contain line at the sides
    for i in tqdm(range(df.shape[0])):
        #somehow the original input is inverted
        img0 = 255 - df.loc[df.index[i]].values.reshape(137, 236).astype(np.uint8)
        #normalize each image by its max val
        img0 = (img0*(255.0/img0.max())).astype(np.uint8)
        
#         _, thresh = cv2.threshold(img0,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
#         ymin,ymax,xmin,xmax = bbox(thresh[5:-5,5:-5] > 80)
#         #cropping may cut too much, so we need to add it back
#         xmin = xmin - 13 if (xmin > 13) else 0
#         ymin = ymin - 10 if (ymin > 10) else 0
#         xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH
#         ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT
#         img = img0[ymin:ymax,xmin:xmax]
#         #remove lo intensity pixels as noise
#         img[img < 28] = 0
#         lx, ly = xmax-xmin,ymax-ymin
#         l = max(lx,ly) + pad
#         #make sure that the aspect ratio is kept in rescaling
#         img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')
        resized_roi = cv2.resize(img0, (size, size),interpolation=cv2.INTER_AREA)
        resized[df.index[i]] = resized_roi.reshape(-1)
    resized = pd.DataFrame(resized).T
    return resized

划分数据集方面sklearn也很强大

x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train,
                                                    test_size=0.1, random_state=42)

也有kfold的，原理差不多，都挺好用

当训练的数据过大时，内存难以一次加载完成就需要del了

del x_train, x_test, y_train, y_test
gc.collect()

gc的库需要单独导入，不然就是表面删除了