文章介绍了一个名为CLRNet的深度学习模型，该模型结合了ResNet块和卷积LSTM层，用于视频数据的处理。CLRNet提供了不同结构的构建块，如基本块和瓶颈块，并支持不同版本的残差单元。此外，文章还包含了数据预处理、训练序列生成以及模型训练和验证的实现。模型在多个数据集上进行了训练和验证，展示了其在图像识别任务中的应用。
pip install tensorflow==2.1pip install tensorflow-gpu==2.1.0pip install keras==2.3.1# CLRNet_convlstm.py
from __future__ import division
import six
from tensorflow.python.keras import backend as K, Model, Input
from tensorflow.python.keras.layers import BatchNormalization, Activation, Dense, Dropout, ConvLSTM2D, add, GlobalAveragePooling3D, Reshape, GlobalMaxPooling3D, MaxPooling3D
from tensorflow.python.keras.regularizers import l2
def _bn_relu(x, bn_name=None, relu_name=None):
"""Helper to build a BN -> relu block
"""
norm = BatchNormalization(axis=CHANNEL_AXIS, name=bn_name)(x)
return Activation("relu", name=relu_name)(norm)
def _bn_relu2(x, bn_name=None, relu_name=None):
"""Helper to build a BN -> relu block
"""
norm = BatchNormalization(axis=CHANNEL_AXIS, name=bn_name)(x)
norm=GlobalAveragePooling3D()(norm)
norm=Dropout(0.25)(norm)
norm=Dense(units=2, activation='softmax',kernel_initializer="he_normal")(norm)
return norm
def _conv_bn_relu(**conv_params):
"""Helper to build a conv -> BN -> relu residual unit activation function.
This is the original ResNet v1 scheme in https://arxiv.org/abs/1512.03385
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", (1, 1))
dilation_rate = conv_params.setdefault("dilation_rate", (1, 1))
conv_name = conv_params.setdefault("conv_name", None)
bn_name = conv_params.setdefault("bn_name", None)
relu_name = conv_params.setdefault("relu_name", None)
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
padding = conv_params.setdefault("padding", "same")
kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(x):
x = ConvLSTM2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
dilation_rate=dilation_rate,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer,return_sequences=True,
name=conv_name)(x)
return _bn_relu(x, bn_name=bn_name, relu_name=relu_name)
return f
def _bn_relu_conv(**conv_params):
"""Helper to build a BN -> relu -> conv residual unit with full pre-activation
function. This is the ResNet v2 scheme proposed in
http://arxiv.org/pdf/1603.05027v2.pdf
"""
filters = conv_params["filters"]
kernel_size = conv_params["kernel_size"]
strides = conv_params.setdefault("strides", (1, 1))
dilation_rate = conv_params.setdefault("dilation_rate", (1, 1))
conv_name = conv_params.setdefault("conv_name", None)
bn_name = conv_params.setdefault("bn_name", None)
relu_name = conv_params.setdefault("relu_name", None)
kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal")
padding = conv_params.setdefault("padding", "same")
kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4))
def f(x):
activation = _bn_relu(x, bn_name=bn_name, relu_name=relu_name)
return ConvLSTM2D(filters=filters, kernel_size=kernel_size,
strides=strides, padding=padding,
dilation_rate=dilation_rate,
kernel_initializer=kernel_initializer,
kernel_regularizer=kernel_regularizer,return_sequences=True,
name=conv_name)(activation)
return f
def _shortcut(input_feature, residual, conv_name_base=None, bn_name_base=None):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input_feature)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input_feature
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
print('reshaping via a convolution...')
if conv_name_base is not None:
conv_name_base = conv_name_base + '1'
shortcut = ConvLSTM2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001),return_sequences=True,
name=conv_name_base)(input_feature)
if bn_name_base is not None:
bn_name_base = bn_name_base + '1'
shortcut = BatchNormalization(axis=CHANNEL_AXIS,
name=bn_name_base)(shortcut)
return add([shortcut, residual])
def _residual_block(block_function, filters, blocks, stage,
transition_strides=None, transition_dilation_rates=None,
dilation_rates=None, is_first_layer=False, dropout=None,
residual_unit=_bn_relu_conv):
"""Builds a residual block with repeating bottleneck blocks.
stage: integer, current stage label, used for generating layer names
blocks: number of blocks 'a','b'..., current block label, used for generating
layer names
transition_strides: a list of tuples for the strides of each transition
transition_dilation_rates: a list of tuples for the dilation rate of each
transition
"""
if transition_dilation_rates is None:
transition_dilation_rates = [(1, 1)] * blocks
if transition_strides is None:
transition_strides = [(1, 1)] * blocks
if dilation_rates is None:
dilation_rates = [1] * blocks
def f(x):
for i in range(blocks):
is_first_block = is_first_layer and i == 0
x = block_function(filters=filters, stage=stage, block=i,
transition_strides=transition_strides[i],
dilation_rate=dilation_rates[i],
is_first_block_of_first_layer=is_first_block,
dropout=dropout,
residual_unit=residual_unit)(x)
return x
return f
def _block_name_base(stage, block):
"""Get the convolution name base and batch normalization name base defined by
stage and block.
If there are less than 26 blocks they will be labeled 'a', 'b', 'c' to match the
paper and keras and beyond 26 blocks they will simply be numbered.
"""
if block < 27:
block = '%c' % (block + 97) # 97 is the ascii number for lowercase 'a'
conv_name_base = 'res' + str(stage) + str(block) + '_branch'
bn_name_base = 'bn' + str(stage) + str(block) + '_branch'
return conv_name_base, bn_name_base
def basic_block(filters, stage, block, transition_strides=(1, 1),
dilation_rate=(1, 1), is_first_block_of_first_layer=False, dropout=None,
residual_unit=_bn_relu_conv):
"""Basic 3 X 3 convolution blocks for use on resnets with layers <= 34.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
"""
def f(input_features):
conv_name_base, bn_name_base = _block_name_base(stage, block)
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
x = ConvLSTM2D(filters=filters, kernel_size=(3, 3),
strides=transition_strides,
dilation_rate=dilation_rate,
padding="same",
kernel_initializer="he_normal",
kernel_regularizer=l2(1e-4),return_sequences=True,
name=conv_name_base + '2a')(input_features)
else:
x = residual_unit(filters=filters, kernel_size=(3, 3),
strides=transition_strides,
dilation_rate=dilation_rate,
conv_name_base=conv_name_base + '2a',
bn_name_base=bn_name_base + '2a')(input_features)
if dropout is not None:
x = Dropout(dropout)(x)
x = residual_unit(filters=filters, kernel_size=(3, 3),
conv_name_base=conv_name_base + '2b',
bn_name_base=bn_name_base + '2b')(x)
return _shortcut(input_features, x)
return f
def bottleneck(filters, stage, block, transition_strides=(1, 1),
dilation_rate=(1, 1), is_first_block_of_first_layer=False, dropout=None,
residual_unit=_bn_relu_conv):
"""Bottleneck architecture for > 34 layer resnet.
Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf
Returns:
A final conv layer of filters * 4
"""
def f(input_feature):
conv_name_base, bn_name_base = _block_name_base(stage, block)
if is_first_block_of_first_layer:
# don't repeat bn->relu since we just did bn->relu->maxpool
x = ConvLSTM2D(filters=filters, kernel_size=(3, 3),
strides=transition_strides,
dilation_rate=dilation_rate,
padding="same",
kernel_initializer="he_normal",return_sequences=True,
kernel_regularizer=l2(1e-4),
name=conv_name_base + '2a')(input_feature)
else:
x = residual_unit(filters=filters, kernel_size=(1, 1),
strides=transition_strides,
dilation_rate=dilation_rate,
conv_name_base=conv_name_base + '2a',
bn_name_base=bn_name_base + '2a')(input_feature)
if dropout is not None:
x = Dropout(dropout)(x)
x = residual_unit(filters=filters, kernel_size=(3, 3),
conv_name_base=conv_name_base + '2b',
bn_name_base=bn_name_base + '2b')(x)
if dropout is not None:
x = Dropout(dropout)(x)
x = residual_unit(filters=filters * 4, kernel_size=(1, 1),
conv_name_base=conv_name_base + '2c',
bn_name_base=bn_name_base + '2c')(x)
return _shortcut(input_feature, x)
return f
def _handle_dim_ordering():
global ROW_AXIS
global COL_AXIS
global CHANNEL_AXIS
if K.image_data_format() == 'channels_last':
ROW_AXIS = 2
COL_AXIS = 3
CHANNEL_AXIS = 4
else:
CHANNEL_AXIS = 2
ROW_AXIS = 3
COL_AXIS = 4
def _string_to_function(identifier):
if isinstance(identifier, six.string_types):
res = globals().get(identifier)
if not res:
raise ValueError('Invalid {}'.format(identifier))
return res
return identifier
def CLRNet(input_shape=None, classes=10, block='bottleneck', residual_unit='v2',
repetitions=None, initial_filters=64, activation='softmax', include_top=True,
input_tensor=None, dropout=None, transition_dilation_rate=(1, 1),
initial_strides=(2, 2), initial_kernel_size=(7, 7), initial_pooling='max',
final_pooling=None, top='classification'):
"""Builds a custom ResNet like architecture. Defaults to CLRNet50 v2.
Args:
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` dim ordering)
or `(3, 224, 224)` (with `channels_first` dim ordering).
It should have exactly 3 dimensions,
and width and height should be no smaller than 8.
E.g. `(224, 224, 3)` would be one valid value.
classes: The number of outputs at final softmax layer
block: The block function to use. This is either `'basic'` or `'bottleneck'`.
The original paper used `basic` for layers < 50.
repetitions: Number of repetitions of various block units.
At each block unit, the number of filters are doubled and the input size
is halved. Default of None implies the CLRNet50v2 values of [3, 4, 6, 3].
residual_unit: the basic residual unit, 'v1' for conv bn relu, 'v2' for bn relu
conv. See [Identity Mappings in
Deep Residual Networks](https://arxiv.org/abs/1603.05027)
for details.
dropout: None for no dropout, otherwise rate of dropout from 0 to 1.
Based on [Wide Residual Networks.(https://arxiv.org/pdf/1605.07146) paper.
transition_dilation_rate: Dilation rate for transition layers. For semantic
segmentation of images use a dilation rate of (2, 2).
initial_strides: Stride of the very first residual unit and MaxPooling2D call,
with default (2, 2), set to (1, 1) for small images like cifar.
initial_kernel_size: kernel size of the very first convolution, (7, 7) for
imagenet and (3, 3) for small image datasets like tiny imagenet and cifar.
See [ResNeXt](https://arxiv.org/abs/1611.05431) paper for details.
initial_pooling: Determine if there will be an initial pooling layer,
'max' for imagenet and None for small image datasets.
See [ResNeXt](https://arxiv.org/abs/1611.05431) paper for details.
final_pooling: Optional pooling mode for feature extraction at the final
model layer when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
top: Defines final layers to evaluate based on a specific problem type. Options
are 'classification' for ImageNet style problems, 'segmentation' for
problems like the Pascal VOC dataset, and None to exclude these layers
entirely.
Returns:
The keras `Model`.
"""
if activation not in ['softmax', 'sigmoid', None]:
raise ValueError('activation must be one of "softmax", "sigmoid", or None')
if activation == 'sigmoid' and classes != 1:
raise ValueError('sigmoid activation can only be used when classes = 1')
if repetitions is None:
repetitions = [3, 4, 6, 3]
_handle_dim_ordering()
if len(input_shape) != 4:
raise Exception("Input shape should be a tuple (frames,nb_channels, nb_rows, nb_cols)")
if block == 'basic':
block_fn = basic_block
elif block == 'bottleneck':
block_fn = bottleneck
elif isinstance(block, six.string_types):
block_fn = _string_to_function(block)
else:
block_fn = block
if residual_unit == 'v2':
residual_unit = _bn_relu_conv
elif residual_unit == 'v1':
residual_unit = _conv_bn_relu
elif isinstance(residual_unit, six.string_types):
residual_unit = _string_to_function(residual_unit)
else:
residual_unit = residual_unit
# Permute dimension order if necessary
if K.image_data_format() == 'channels_first':
input_shape = (input_shape[1], input_shape[2], input_shape[0])
img_input = Input(shape=input_shape, tensor=input_tensor)
x = _conv_bn_relu(filters=initial_filters, kernel_size=initial_kernel_size,
strides=initial_strides)(img_input)
if initial_pooling == 'max':
# x = MaxPooling3D(pool_size=(3, 3, 3), strides=initial_strides, padding="same")(x)
x = MaxPooling3D(pool_size=(1, 3, 3), strides=None, padding="same")(x)
block = x
filters = initial_filters
for i, r in enumerate(repetitions):
transition_dilation_rates = [transition_dilation_rate] * r
transition_strides = [(1, 1)] * r
if transition_dilation_rate == (1, 1):
transition_strides[0] = (2, 2)
block = _residual_block(block_fn, filters=filters,
stage=i, blocks=r,
is_first_layer=(i == 0),
dropout=dropout,
transition_dilation_rates=transition_dilation_rates,
transition_strides=transition_strides,
residual_unit=residual_unit)(block)
filters *= 2
# Last activation
x = _bn_relu2(block)
# Classifier block
if include_top and top is 'classification':
x = GlobalAveragePooling3D()(x)
x = Dense(units=classes, activation=activation,
kernel_initializer="he_normal")(x)
elif include_top and top is 'segmentation':
x = ConvLSTM2D(classes, (1, 1), activation='linear', padding='same',return_sequences=True)(x)
if K.image_data_format() == 'channels_first':
channel, row, col = input_shape
else:
row, col, channel = input_shape
x = Reshape((row * col, classes))(x)
x = Activation(activation)(x)
x = Reshape((row, col, classes))(x)
elif final_pooling == 'avg':
x = GlobalAveragePooling3D()(x)
elif final_pooling == 'max':
x = GlobalMaxPooling3D()(x)
model = Model(inputs=img_input, outputs=x)
return model
def CLRNet18(input_shape, classes):
"""CLRNet with 18 layers and v2 residual units
"""
return CLRNet(input_shape, classes, basic_block, repetitions=[2, 2, 2, 2])
def CLRNet34(input_shape, classes):
"""CLRNet with 34 layers and v2 residual units
"""
return CLRNet(input_shape, classes, basic_block, repetitions=[3, 4, 6, 3])
def CLRNet50(input_shape, classes):
"""CLRNet with 50 layers and v2 residual units
"""
return CLRNet(input_shape, classes, bottleneck, repetitions=[3, 4, 6, 3])
def CLRNet101(input_shape, classes):
"""CLRNet with 101 layers and v2 residual units
"""
return CLRNet(input_shape, classes, bottleneck, repetitions=[3, 4, 23, 3])
def CLRNet152(input_shape, classes):
"""CLRNet with 152 layers and v2 residual units
"""
return CLRNet(input_shape, classes, bottleneck, repetitions=[3, 8, 36, 3])pip install --upgrade natsort# Utility_functions.py
import natsort
import random
import glob
import os
#from tensorflow.python.keras.utils import to_categorical
from tensorflow.keras.utils import to_categorical
from tensorflow.python.keras import layers
from sklearn.utils import class_weight
import numpy as np
random.seed(32)
import pandas as pd
from tensorflow.python.keras.callbacks import Callback
def downsample_withoutshuffle(f_list, fpvpb, no_frames):
f_list = natsort.natsorted(f_list)
_total = int(no_frames / fpvpb) # 20/5
_tempList = []
# print("Size:",_total)
if len(f_list) > _total: # if list smaller than required frames
for i in range(0, _total):
_range = len(f_list) - fpvpb
if _range == 0:
return f_list
x = random.randrange(_range)
for j in range(0, fpvpb):
# print(os.path.splitext(os.path.basename(f_list[x+j]))[0],end=",")
_tempList.append(f_list[x + j])
# print()
return _tempList
else:
return f_list
def create_sequence(dirs, fpvpb, no_frames):
random.seed(35)
count_real = 0
count_fake = 0
folders=[]
for directory in dirs:
folders += [i for i in sorted(glob.glob(os.path.join(directory, '*', "*")))]
random.shuffle(folders)
total_folders = len(folders)
X = {}
y = []
print('Total Video Folders Found (Real + Fake):', total_folders)
pre_folder = -1
i = 0
_downsampled = []
while i < total_folders:
folder = random.choice(folders)
dir_name = os.path.dirname(folder)
folder_name = os.path.basename(dir_name)
# print(folder_name)
_downsampled = [x for x in sorted(glob.glob(os.path.join(folder, "*")))][0:no_frames]
# _downsampled=downsample_withoutshuffle([x for x in sorted(glob.glob(os.path.join(folder,"*")))],fpvpb,no_frames)
if folder_name == 'real':
if pre_folder == 1:
continue
y.append(1)
count_real += len(_downsampled)
pre_folder = 1
elif folder_name == 'fake':
if pre_folder == 0:
continue
y.append(0)
count_fake += len(_downsampled)
pre_folder = 0
else:
print("Directory names should be 'real' for + (1) and 'fake' for label (0)")
exit(0)
# X[str(i)]=[x for x in sorted(glob.glob(os.path.join(folder,"*")))]
X[str(i)] = _downsampled
folders.remove(folder)
i += 1
print('Real Frames:', count_real, 'Fake Frames:', count_fake)
labels = []
for i in range(0, count_real):
labels.append(1)
for i in range(0, count_fake):
labels.append(0)
y_ints = [v.argmax() for v in to_categorical(labels)]
### class_weights = class_weight.compute_class_weight('balanced', np.unique(y_ints), y_ints)
class_weights = class_weight.compute_class_weight(class_weight='balanced',classes=np.unique(y_ints),y= y_ints)
print('Class Weights:', class_weights)
return X, y, class_weights
class FrozenBatchNormalization(layers.BatchNormalization):
def call(self, inputs, training=None):
return super().call(inputs=inputs, training=False)
def FreezeBatchNormalization(is_training,top_k_layers,model):
if is_training == True:
_bottom_layers = model.layers[:-top_k_layers]
_top_layers = model.layers[-top_k_layers:]
elif is_training == False:
_bottom_layers = model.layers
_top_layers = []
for _layer in _bottom_layers:
_layer.trainable = False
if 'batch_normalization' in _layer.name:
# print('Freezing BN layers ... {}'.format(_layer.name))
_layer = FrozenBatchNormalization
for _layer in _top_layers:
_layer.trainable = True
if 'batch_normalization' in _layer.name:
# print('Unfreezing BN layers ... {}'.format(_layer.name))
_layer = layers.BatchNormalization
layers_df = [(layer, layer.name, layer.trainable) for layer in model.layers]
df=pd.DataFrame(layers_df, columns=['Layer Type', 'Layer Name', 'Layer Trainable'])
return model,df
class AdditionalValidationSets(Callback):
def __init__(self, validation_gen_sets,filename, verbose=1, batch_size=None):
"""
:param validation_sets:
a list of 3-tuples (validation_data, validation_targets, validation_set_name)
or 4-tuples (validation_data, validation_targets, sample_weights, validation_set_name)
:param verbose:
verbosity mode, 1 or 0
:param batch_size:
batch size to be used when evaluating on the additional datasets
"""
super(AdditionalValidationSets, self).__init__()
self.validation_gen_sets = validation_gen_sets
# for validation_set in self.validation_sets:
# if len(validation_set) not in [2, 3]:
# raise ValueError()
self.epoch = []
self.history = {}
self.verbose = verbose
self.batch_size = batch_size
self.filename = filename
self.previous_data=""
self.save_previous=False
def on_train_begin(self, logs=None):
self.epoch = []
self.history = {}
if os.path.exists(self.filename):
self.save_previous=True
with open(self.filename, 'r') as file:
self.previous_data = file.read()
def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
self.epoch.append(epoch)
# line=str(epoch)+","
# record the same values as History() as well
for k, v in logs.items():
self.history.setdefault(k, []).append(v)
# evaluate on the additional validation sets
for validation_gen_set in self.validation_gen_sets:
if len(validation_gen_set) == 2:
validation_gen, validation_set_name = validation_gen_set
# sample_weights = None
# elif len(validation_set) == 4:
# validation_data, validation_targets, sample_weights, validation_set_name = validation_set
# else:
# raise ValueError()
results = self.model.evaluate_generator(validation_gen,
verbose=self.verbose)
# sample_weight=sample_weights,
# batch_size=self.batch_size)
for i, result in enumerate(results):
if i == 0:
valuename = validation_set_name + '_loss'
else:
valuename = str(validation_set_name) + '_acc'#acc + str(self.model.metrics[i-1])
self.history.setdefault(valuename, []).append(result)
line=""
previous=0
if self.save_previous==False:
line="epoch,"
dic=self.history
for item in dic:
line+=item+","
line=line[:-1]+"\n"
else:
line=self.previous_data
previous=len(line.splitlines())
for i in range(0,len(dic['loss'])):
line+=str(i+1+previous)+","
for item in dic:
line+=str(dic[item][i])+","
line=line[:-1]+"\n"
with open(self.filename, 'w') as the_file:
the_file.write(line)# DFVDSequence.py
#from tensorflow.python.keras.utils import Sequence,to_categorical
from tensorflow.keras.utils import Sequence,to_categorical
from tensorflow.python.keras.preprocessing.image import ImageDataGenerator
from PIL import Image
import numpy as np
import cv2
from copy import deepcopy
# Here, `x_set` is list of path to the images,
# `y_set` are the associated classes,
# 'v' is the no. of videos for each batch
# and fpv is the no. of frames from each video
print('loaded')
class DFVDSequence(Sequence):
def __init__(self, x_set, y_set, v, fpv, image_size=240,data_augmentation=True,training=True):
self.x, self.y = x_set, y_set
self.batch_size = fpv * v
self.count = 0
self.video_iterator = 0
self.frame_iterator = 0
self.v = v
self.fpv = fpv
self.dataset_size = sum([len(value) for key, value in self.x.items()])
self.frame_counter = 0
self.image_size = image_size
self.data_augmentation=data_augmentation
self.classes = []
self.pre_classes = []
# self.class_list=[]
self.on_epoch_end()
self.training=training
def __len__(self):
return int(np.ceil(self.dataset_size / float(self.batch_size)))
def resize_img(self, img, basewidth=240):
wpercent = (basewidth / float(img.size[0]))
hsize = int((float(img.size[1]) * float(wpercent)))
# img = img.resize((basewidth,hsize), Image.ANTIALIAS)
img = img.resize((basewidth, basewidth), Image.ANTIALIAS)
return img
def img_transf(self, img):
# img -= img.mean()
# img /= np.maximum(img.std(), 1/image_size**2) #prevent /0
return img
def __getitem__(self, idx):
batch_x = []
batch_y = []
video_idx = 0
_iter = 0
while _iter < self.v:
_iter += 1
if self.frame_counter >= self.dataset_size:
self.on_epoch_end()
print('*', end=",")
_iter -= 1
continue
frames2read = self.x[str(self.video_iterator)][self.frame_iterator:self.frame_iterator + self.fpv]
temp_frames = []
if self.data_augmentation== True:
img_gen = ImageDataGenerator(rotation_range=30,
samplewise_center=True,
samplewise_std_normalization=True,
# width_shift_range=0.2,
# height_shift_range=0.2,
# rescale=1./255,
brightness_range=[0.7,1.0],
channel_shift_range=50.0,
# shear_range=0.1,
zoom_range=0.2,
horizontal_flip=True,
# vertical_flip = True,
fill_mode='nearest')
transform_param = img_gen.get_random_transform(img_shape=(self.image_size, self.image_size, 3), seed=None)
# print(transform_param)
if len(frames2read) >= self.fpv:
for frame in frames2read:
_image = cv2.imread(frame)
_image = cv2.cvtColor(_image, cv2.COLOR_BGR2RGB).astype('float64')
_image = cv2.resize(_image, (self.image_size, self.image_size))
if self.data_augmentation == True:
_image = img_gen.apply_transform(_image, transform_param)
_image /= 255
temp_frames.append(np.asarray(_image))
batch_x.append(temp_frames)
batch_y.append(self.y[self.video_iterator])
self.pre_classes.append(self.y[self.video_iterator])
else:
_iter -= 1
self.video_iterator += 1
self.frame_counter += len(frames2read)
if self.video_iterator % len(self.x) == 0:
self.video_iterator = 0
self.frame_iterator += len(frames2read)
video_idx += 1
if self.video_iterator % len(self.x) == 0:
self.video_iterator = 0
if len(batch_y) > 0:
batch_y = to_categorical(batch_y, 2)
batch_x = np.array(batch_x)
if self.training == True:
indices = np.arange(batch_x.shape[0])
np.random.shuffle(indices)
batch_x = batch_x[indices]
batch_y = batch_y[indices]
return (batch_x, batch_y)
def on_epoch_end(self):
""" Method called at the end of every epoch. """
self.count = 0
self.video_iterator = 0
self.frame_iterator = 0
self.frame_counter = 0
self.classes=deepcopy(self.pre_classes)
self.pre_classes=[]
returnloaded
CLRNET_DF+FS.ipynb
#Import Python Packages
from tensorflow.python.keras.backend import clear_session
from tensorflow.python.keras.callbacks import ModelCheckpoint, CSVLogger
from tensorflow.python.keras.utils.layer_utils import print_summary
from tensorflow.python.keras.optimizers import adam_v2
from tensorflow.python.keras.models import load_model
import numpy as np
# from CLRNet_convlstm import CLRNet
# from Utility_functions import create_sequence,FreezeBatchNormalization
from datetime import datetime as dt
# from DFVDSequence import DFVDSequence
import matplotlib.pyplot as plt
print("yolo")
from sklearn.metrics import classification_report
import pandas as pd
# import shutil
from tensorflow.python.keras import layers
# from src.xception_convlstm import Xception
# print(cv2.__version__)
# import random
# random.seed(32)
dataset_dir='DeepFakeDatasetReal'
# from src.cl_basic import cl_basic
import ipykernel
import os
from sklearn.preprocessing import MinMaxScaler
import tensorflow as tf
from tensorflow.keras import backend as K
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID";
# Choose GPU NUMBERS [0, 1, 2, 3]
os.environ["CUDA_VISIBLE_DEVICES"]="0"
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
tf.compat.v1.disable_eager_execution() # 保证sess.run()能够正常运行
sess = tf.compat.v1.Session(config=config)
tf.compat.v1.keras.backend.set_session(sess)
sess.run(tf.compat.v1.global_variables_initializer())
yolo
### Load DF+FS Transfer Dataset
train_video_per_batch=20 #40
val_video_per_batch=25
test_video_per_batch=25
train_frames_per_video=10 #total frames
val_frames_per_video=5 #total frames
test_frames_per_video=5 #total frames
frames_per_video_per_batch=5 #frames in one batch
image_size=64 #128
channel=3
train_augumentation=True
val_augumentation=False
test_augumentation=False
X_train,y_train, class_weights_train = create_sequence(['/kaggle/input/datasets15/datasets/DeepFake/train',
'/kaggle/input/datasets15/datasets/FaceSwap/train'],
frames_per_video_per_batch,
train_frames_per_video)
X_val_0,y_val_0,class_weights_val_0=create_sequence(['/kaggle/input/datasets15/datasets/DeepFake/val'],
frames_per_video_per_batch,
val_frames_per_video)
X_val_1,y_val_1,class_weights_val_1=create_sequence(['/kaggle/input/datasets15/datasets/FaceSwap/val'],
frames_per_video_per_batch,
val_frames_per_video)
train_it=DFVDSequence(X_train,y_train,train_video_per_batch,
frames_per_video_per_batch,image_size,
train_augumentation,True)
val_it_0=DFVDSequence(X_val_0,y_val_0,val_video_per_batch,
frames_per_video_per_batch,image_size,
val_augumentation,False)
val_it_1=DFVDSequence(X_val_1,y_val_1,val_video_per_batch,
frames_per_video_per_batch,image_size,
val_augumentation,False)
fig=plt.figure(figsize=(10, 100))
columns = 4
rows = 23
print("Training")
x=1
temp=train_it.__getitem__(1)
fig.add_subplot(rows, columns, x)
print("Min:",np.array(temp[0][0,0]).min(),"Max:",np.array(temp[0][0,0]).max())
print("Min:",np.array(temp[0][1,0]).min(),"Max:",np.array(temp[0][1,0]).max())
plt.imshow(np.array(temp[0][0,0]))
fig.add_subplot(rows, columns, x+1)
plt.imshow(np.array(temp[0][1,0]))
print(temp[0].shape,temp[1].shape,train_it.dataset_size,train_it.frame_counter,train_it.__len__(),temp[1][0],temp[1][1])
# x+=2
plt.title("Training Set Examples")
plt.show()
train_it.on_epoch_end()
fig=plt.figure(figsize=(10, 100))
columns = 4
rows = 23
print("Validation - 0")
x=1
temp=val_it_0.__getitem__(1)
fig.add_subplot(rows, columns, x)
print("Min:",np.array(temp[0][0,0]).min(),"Max:",np.array(temp[0][0,0]).max())
print("Min:",np.array(temp[0][1,0]).min(),"Max:",np.array(temp[0][1,0]).max())
plt.imshow(np.array(temp[0][0,0]),)
fig.add_subplot(rows, columns, x+1)
plt.imshow(np.array(temp[0][1,0]))
print(temp[0].shape,temp[1].shape,val_it_0.dataset_size,val_it_0.frame_counter,val_it_0.__len__(),temp[1][0],temp[1][1])
plt.title("Validation - 0 Set Examples")
plt.show()
val_it_0.on_epoch_end()
fig=plt.figure(figsize=(10, 100))
columns = 4
rows = 23
print("Validation - 1")
x=1
temp=val_it_1.__getitem__(1)
fig.add_subplot(rows, columns, x)
print("Min:",np.array(temp[0][0,0]).min(),"Max:",np.array(temp[0][0,0]).max())
print("Min:",np.array(temp[0][1,0]).min(),"Max:",np.array(temp[0][1,0]).max())
plt.imshow(np.array(temp[0][0,0]),)
fig.add_subplot(rows, columns, x+1)
plt.imshow(np.array(temp[0][1,0]))
print(temp[0].shape,temp[1].shape,val_it_1.dataset_size,val_it_1.frame_counter,val_it_1.__len__(),temp[1][0],temp[1][1])
plt.title("Validation - 1 Set Examples")
plt.show()
val_it_1.on_epoch_end()fig=plt.figure(figsize=(10, 100))
columns = 5
rows = 40
print("ALL")
x=1
temp=train_it.__getitem__(1)
fig.add_subplot(rows, columns, x)
plt.imshow(np.array(temp[0][0,0]))
fig.add_subplot(rows, columns, x+1)
plt.imshow(np.array(temp[0][0,1]))
fig.add_subplot(rows, columns, x+2)
plt.imshow(np.array(temp[0][0,2]))
fig.add_subplot(rows, columns, x+3)
plt.imshow(np.array(temp[0][0,3]))
fig.add_subplot(rows, columns, x+4)
plt.imshow(np.array(temp[0][0,4]))
print("Min:",np.array(temp[0][0,0]).min(),"Max:",np.array(temp[0][0,0]).max())
fig.add_subplot(rows, columns, x+5)
plt.imshow(np.array(temp[0][1,0]))
fig.add_subplot(rows, columns, x+6)
plt.imshow(np.array(temp[0][1,1]))
fig.add_subplot(rows, columns, x+7)
plt.imshow(np.array(temp[0][1,2]))
fig.add_subplot(rows, columns, x+8)
plt.imshow(np.array(temp[0][1,3]))
fig.add_subplot(rows, columns, x+9)
plt.imshow(np.array(temp[0][1,4]))
print("Min:",np.array(temp[0][1,0]).min(),"Max:",np.array(temp[0][1,0]).max())
print(temp[0].shape,temp[1].shape,train_it.dataset_size,train_it.frame_counter,train_it.__len__(),temp[1][0],temp[1][1])
plt.title("Training Set Examples")
plt.show()
train_it.on_epoch_end()#Load Model
# is_training = True
# top_k_layers=120
# model,df=FreezeBatchNormalization(is_training,top_k_layers,model)
# with pd.option_context('display.max_rows', None, 'display.max_columns', None): # more options can be specified also
# display(df)
model=CLRNet(input_shape=(frames_per_video_per_batch,image_size, image_size, channel), classes=2, block='bottleneck', residual_unit='v2',
repetitions=[3, 4, 6, 3], initial_filters=64, activation='softmax', include_top=False,
input_tensor=None, dropout=0.25, transition_dilation_rate=(1, 1),
initial_strides=(2, 2), initial_kernel_size=(7, 7), initial_pooling='max',
final_pooling=None, top='classification')
# print_summary(model, line_length=150, positions=None, print_fn=None)
adam_fine = adam_v2.Adam(learning_rate=0.00005, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(loss='binary_crossentropy',
optimizer=adam_fine,
metrics=['acc'])
print_summary(model, line_length=150, positions=None, print_fn=None)#Compiling Model
_id = dt.now().strftime("%y-%m-%d-%H_%M")
save_dir=os.path.join('/kaggle/working/','models','CLRNET_DF+FS','CLRNet','DF+FS',_id)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
filepath=os.path.join(save_dir,"{epoch:02d}-{acc:.2f}"+".hdf5")
checkpoint = ModelCheckpoint(filepath)#, monitor='acc', verbose=2, save_best_only=True, mode='max')
csv_logger = AdditionalValidationSets([(val_it_0,'valDF'),(val_it_1,'valFS')],os.path.join(save_dir,"training.csv"))
callbacks_list = [csv_logger,checkpoint] #Training
model.fit_generator(generator=train_it,epochs=4,
callbacks=callbacks_list,shuffle=False,class_weight=class_weights_train,verbose=1)model = load_model('/kaggle/working/media/data1/sha/models/CLRNET_DF+FS/CLRNet/DF+FS/20-05-28-08_24/98-0.80.hdf5')
model.fit_generator(generator=train_it,epochs=150, initial_epoch=98,
callbacks=callbacks_list,shuffle=False,class_weight=class_weights_train,verbose=1)
扫一扫
6040
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
