随机2

# https://developer.nvidia-china.com/forum.php?mod=viewthread&tid=10722&extra=page%3D1
import sys
import numpy as np
import math

# ====================================

from collections import OrderedDict, namedtuple, defaultdict
from itertools import chain

import tensorflow as tf
print('=======================================')
print(tf.__version__)
print('=======================================')


gpus = tf.config.experimental.list_physical_devices(device_type='GPU')
for gpu in gpus:
    print(gpu)
    tf.config.experimental.set_memory_growth(gpu, True)


class NoMask(tf.keras.layers.Layer):
    def __init__(self, **kwargs):
        super(NoMask, self).__init__(**kwargs)

    def build(self, input_shape):
        # Be sure to call this somewhere!
        super(NoMask, self).build(input_shape)

    def call(self, x, mask=None, **kwargs):
        return x

    def compute_mask(self, inputs, mask):
        return None

def concat_func(inputs, axis=-1, mask=False):
    if not mask:
        inputs = list(map(NoMask(), inputs))
    if len(inputs) == 1:
        return inputs[0]
    else:
        return tf.keras.layers.Concatenate(axis=axis)(inputs)
# =======================================================================

def create_embedding_dict(params,  seq_mask_zero=True, init_embed_dic={}):
    with tf.device("/cpu:0"):
        sparse_embedding = OrderedDict()
        for col in params['speat_feat_cols']:
            trainable = False
            vocb_size, embed_size = params['embedding_vobsize_dict'].get(col), params['embedding_size_dict'].get(col)
            if col  in init_embed_dic:
                init_emb = init_embed_dic.get(col)
                sparse_embedding[col] = tf.keras.layers.Embedding(vocb_size, embed_size,
                                                                  weights=[init_emb],
                                                                  trainable=trainable,
                                                                  name='seq_emb_' + col,
                                                                  mask_zero=seq_mask_zero)
            else:
                # init_emb = tf.keras.initializers.glorot_uniform()
                sparse_embedding[col] = tf.keras.layers.Embedding(vocb_size, embed_size,
                                                                  embeddings_initializer='uniform',
                                                                  trainable=trainable,
                                                                  name='seq_emb_' + col,
                                                                  mask_zero=seq_mask_zero)
        return sparse_embedding

def embedding_lookup(sparse_embedding_dict, inputs_dict, params, to_list=False):
    look_sparse_embedding = OrderedDict()
    for col in params['speat_feat_cols']:
        look_sparse_embedding[col] = sparse_embedding_dict[col](inputs_dict[col])
    if to_list:
        look_sparse_embedding = list(look_sparse_embedding.values())
    return look_sparse_embedding

def MyModel( params={}, init_embed_dic={},):
    inputs_dict = OrderedDict()
    for col in params['speat_feat_cols']:
        inputs_dict[col] = tf.keras.Input(shape=(None,), name=col, dtype='int32')
    transformer_mask = tf.keras.layers.Input(( None, ), dtype='int32', name='transformer_mask')
    inputs_dict['transformer_mask'] = transformer_mask

    inputs_list = list(inputs_dict.values())
    print(inputs_list)
    # seq_mask = tf.expand_dims(transformer_mask, axis=-1, name='seq_mask')
    # transformer_mask = transformer_mask[:, tf.newaxis, tf.newaxis, :]
    embedding_dict = create_embedding_dict(params,  seq_mask_zero=True, init_embed_dic=init_embed_dic)
    keys_emb_list = embedding_lookup(embedding_dict, inputs_dict, params, to_list=True)
    print('keys_emb_list : ', keys_emb_list)
    # (n, t, d)
    input_embedding = concat_func(keys_emb_list, mask=True)
    print('keys_emb : ', input_embedding, input_embedding.shape)
    drop_rate = 0.2
    input_embedding = tf.keras.layers.Dense(params['d_model'], activation='relu', )(input_embedding)
    input_embedding = tf.keras.layers.Dropout (drop_rate)(input_embedding)

    out2 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128, return_sequences=True,
                                                        recurrent_initializer='glorot_uniform',
                                                        #                               recurrent_dropout=0.2,
                                                        dropout=drop_rate,
                                                        ))(input_embedding)
    out2 = NoMask()(out2)
    # out2 += (seq_mask * -1e9)
    output = tf.keras.layers.GlobalMaxPooling1D()(out2)
    output = tf.keras.layers.Dropout(0.5)(output)

    if params['target'][0]=='age':
        output = tf.keras.layers.Dense(10, use_bias=False, activation='softmax',  name='age')(output)
        model = tf.keras.models.Model(inputs=inputs_list, outputs=[output])
    elif params['target'][0]=='gender':
        # output = tf.keras.layers.Dense(2, use_bias=False, activation='softmax', name='gender')(output)
        output = tf.keras.layers.Dense(1, use_bias=False, activation='sigmoid', name='gender')(output)
        model = tf.keras.models.Model(inputs=inputs_list, outputs=[output])
    else:
        age_output = tf.keras.layers.Dense(10, use_bias=False, activation='softmax', name='age')(output)
        gender_output = tf.keras.layers.Dense(2, use_bias=False, activation='softmax', name='gender')(output)
        model = tf.keras.models.Model(inputs=inputs_list, outputs=[age_output, gender_output])
    return model


class ModelHepler:
    def __init__(self, params):
        self.params = params
        self.callback_list = []
        print('Bulid Model...')
        self.pre_train_embedding_dict = {}
        print('embedding size is : ', sum([v for k, v in self.params['embedding_size_dict'].items() if k in self.params['speat_feat_cols']]))
        self.create_model()

    def create_model(self):
        model = MyModel(params=self.params, init_embed_dic=self.pre_train_embedding_dict, )
        lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
            initial_learning_rate=3e-4,
            decay_steps=200,
            decay_rate=0.996,
            staircase=False,
        )
        optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)
        # optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
#         learning_rate = CustomSchedule(params['d_model'], warmup_steps=2000)
#         optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)

        if self.params['target'][0]=='age':
            model.compile(
                optimizer=optimizer,
                loss={
                    # 'age':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.1),
                    'age': tf.keras.losses.SparseCategoricalCrossentropy(),

                },
                metrics=['accuracy'],
            )
        elif self.params['target'][0]=='gender':
            model.compile(
                optimizer=optimizer,
                loss={
                    # 'gender':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.1),
                    'gender':tf.keras.losses.BinaryCrossentropy(),
                    # 'gender':tf.keras.losses.CategoricalCrossentropy()
                },
                metrics=['accuracy'],
            )
        else:
            model.compile(
                optimizer=optimizer,
                loss={
                  'age':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.1),
                  'gender': tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.1),
                },
                metrics=['accuracy'],
            )
        model.summary()
        self.model =  model

    def fit(self, train_data, validation_data, is_estimator=False):
        print('Train...')
        if is_estimator:

            estimator = tf.keras.estimator.model_to_estimator(self.model, model_dir='../save_model_dir/estimator_0705/')
            # estimator = tf.keras.estimator.model_to_estimator(self.model, keras_model_path='../save_model_dir/estimator_0705/')
            estimator.train(train_data, steps=5)
            print('==== train done ! ======')
            res = estimator.evaluate(validation_data, steps=1 )
            print(res)
            print('==== val done ! ======')
        else:
            model_hist = self.model.fit(
                train_data(),
                epochs=self.params['epochs'],
                verbose=1,
                steps_per_epoch=params['train_steps'],
                # callbacks=self.callback_list,
                validation_data=validation_data(),
                validation_steps=params['val_steps'],
                shuffle=True,
            )
            for k, v in model_hist.history.items():
                print(k, v)


# ================  params =========================
params = {
    'target':['age'],
    'epochs':5,
    'batch_size':128,
    'hist_max_len':100,
    'is_train':True,
    'do_delete':True,
    'continue_train':False,

    'min_freq': 0,
    'speat_feat_cols': ['creative_id', 'advertiser_id',  'time'],
    'pre_train_cols': [ ],

    'example_nums':128*100
}
params['time_cut_len'] = 100
params['processed_data_dir'] = '../processed_data/minfreq-{}'.format(params['min_freq'])
params['pre_train_embed_dir'] = '../pre_trian_emb/gensim/minfreq-{}-cut-{}-sg1-window-10-sample-1e-5'.format(
    params['min_freq'], params['time_cut_len'])

embedding_vobsize_dict = {
    'creative_id': 4445720 + 1,
    'ad_id': 3812202 + 1,
    'product_id': 44315 + 1,
    'product_category': 18 + 1,
    'advertiser_id': 62965 + 1,
    'industry': 336 + 1,
    'time':91+1,
}
embedding_size_dict = {
    'creative_id': 128,
    'ad_id': 128,
    'product_id': 64,
    'product_category': 64,
    'advertiser_id': 128,
    'industry': 64,
    'time':64
}

params['embedding_size_dict'] = embedding_size_dict
params['embedding_vobsize_dict'] = embedding_vobsize_dict

params['d_model'] = sum([v for k, v in params['embedding_size_dict'].items() if k in params['speat_feat_cols']])
print('input embedding size is :', params['d_model'])

# ================================================================================
print('Train dataset...')

record_defaults = ['', '', '', 0, 0]
def decode_features_line(line):
    items = tf.io.decode_csv(line, record_defaults, field_delim=';')
    features = []
    for i, col in enumerate(params['speat_feat_cols']):
        t = tf.strings.split([items[i]], sep=',').values
        t = tf.strings.to_number(t, tf.int32)
        features.append(t)
    return features

def _convert_features_to_dict(*el):
    base_list = params['speat_feat_cols']
    dicto = dict()
    for i in range(len(base_list)):
        dicto[base_list[i]] = el[i]
    # add transformer mask
    dicto['transformer_mask'] = tf.cast(tf.equal(el[0], 0), dtype=tf.int32)
    # dicto['dynamic_rnn_len'] = tf.reduce_sum(tf.cast(tf.not_equal(el[0], 0), dtype=tf.int32), axis=-1)
    return dicto

def decode_labels_line(line):
    items = tf.io.decode_csv(line, record_defaults, field_delim=';')
    labels = []
    i = -2
    for col in params['target']:
        labels.append(items[i])
        i+=1
    # label = tf.one_hot(items[-1], depth=10)
    return labels

def _convert_labels_to_dict(*el):
    base_list = params['target']
    dicto = dict()
    for i in range(len(base_list)):
        dicto[base_list[i]] = el[i]
    return dicto

def input_fn(mode=None):
    tfdata_train_file_path = '../processed_data/tfdata/train.csv'
    data_set = tf.data.TextLineDataset(tfdata_train_file_path,  )
    features_data_set = data_set.map(decode_features_line)
    features_data_set = features_data_set.padded_batch(batch_size=params['batch_size'],
                                                       padded_shapes=(
                                                        tf.TensorShape([None,]),
                                                        tf.TensorShape([None,]),
                                                        tf.TensorShape([None,])
                                                        )
                                                       )
    features_data_set = features_data_set.map(map_func=_convert_features_to_dict)

    label_data_set = data_set.map(decode_labels_line)
    label_data_set = label_data_set.batch(batch_size=params['batch_size'])
    label_data_set = label_data_set.map(_convert_labels_to_dict)
    final_data_set = tf.data.Dataset.zip((features_data_set, label_data_set))
    return final_data_set

params['train_steps'] = math.floor(params['example_nums'] / params['batch_size'] * 0.8)
params['val_steps'] = math.floor(params['example_nums'] / params['batch_size'] * 0.2)
print(params['train_steps'], params['val_steps'])

def train_input_fn():
    final_data_set = input_fn()
    train_data_set = final_data_set.take(params['train_steps']).cache().shuffle(100000)
    return train_data_set

def val_input_fn():
    final_data_set = input_fn()
    val_data_set = final_data_set.skip(params['train_steps']).take(params['val_steps']).cache()
    return val_data_set


model_hepler = ModelHepler(params)
model_hepler.fit(train_data=train_input_fn, validation_data=val_input_fn, )