deepFM model

# ========================================================================
主要知识点：
1.  torch-> torch.data(DataLoader,Dataset, TensorDataset<切分完训练测试集后，合并特征和标签，组成一组数据)
    -> torch.nn -> F, optim -> summary, Model -> roc_auc_score

2.  trn_x, trn_y = train.drop(columns='Label').values, train['Label'].values
    TensorDataset<切分完训练测试集后，合并特征和标签，组成一组数据

3.  dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=32)

4.  可学习参数，需要用nn.Parameter进行定义
    self.w2 = nn.Parameter(torch.rand([fea_num, latent_dim]))

5.  self.dnn_network = nn.ModuleList([nn.Linear(layer[0], layer[1]) 
            for layer in list(zip(hidden_units[:-1],hidden_units[1:]))])

6.  for linear in self.dnn_network:
            x = linear(x)
            x = F.relu(x)
        x = self.dropout(x)

7.  loss_func = nn.BCELoss()
    optimizer = optim.Adam(params=model.parameters(), lr=0.001)

8.  self.embed_layers = nn.ModuleDict({
            'embed_' + str(i):nn.Embedding(num_embeddings=feat['feat_num'],embedding_dim=feat['embed_dim'])
            for i, feat in enumerate(self.sparse_feature_cols)
        })  # 每个离散特征单独embedding

# ========================================================================
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm

import torch
from torch.utils.data import DataLoader, Dataset, TensorDataset
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from torchkeras import summary, Model
from sklearn.metrics import roc_auc_score

import datetime
import warnings
warnings.filterwarnings('ignore')

file_path = 'T:\\FutureAi\\深度学习概念思维导图\\AI-RecommenderSystem-master\\DeepFM\\data\\'

def prepared_data(file_path):
    train = pd.read_csv(file_path + 'train_set.csv')
    val = pd.read_csv(file_path + 'val_set.csv')
    test = pd.read_csv(file_path + 'test_set.csv')

    trn_x, trn_y = train.drop(columns='Label').values, train['Label'].values
    val_x, val_y = val.drop(columns='Label').values, val['Label'].values
    test_x = test.values

    fea_col = np.load(file_path + 'fea_col.npy',allow_pickle=True)
    return fea_col, (trn_x, trn_y),(val_x, val_y), test_x

fea_cols, (trn_x, trn_y), (val_x, val_y), test_x = prepared_data(file_path)

dl_train_dataset = TensorDataset(torch.tensor(trn_x).float(), torch.tensor(trn_y).float())
dl_val_dataset = TensorDataset(torch.tensor(val_x).float(), torch.tensor(val_y).float())

dl_train = DataLoader(dl_train_dataset, shuffle=True, batch_size=8)
dl_val = DataLoader(dl_val_dataset, shuffle=True, batch_size=8)

dl_train
len(dl_train)

for x, y in iter(dl_train):
    print(x.shape, y)
    break


class FM(nn.Module):
    def __init__(self, latent_dim, fea_num):
        super(FM, self).__init__()
        self.latent_dim = latent_dim

        self.w0 = nn.Parameter(torch.zeros([1,]))
        self.w1 = nn.Parameter(torch.rand([fea_num, 1]))
        self.w2 = nn.Parameter(torch.rand([fea_num, latent_dim]))

    def forward(self, inputs):
        first_order = self.w0 + torch.mm(inputs, self.w1)
        second_order = 1/2 * torch.sum(torch.pow(torch.mm(inputs, self.w2),2) 
                            - torch.mm(torch.pow(inputs,2), torch.pow(self.w2, 2)),
                            dim=1,
                            keepdim=True)
        return first_order + second_order

class Dnn(nn.Module):
    def __init__(self, hidden_units, dropout=0.):
        super(Dnn, self).__init__()
        self.dnn_network = nn.ModuleList(
            [nn.Linear(layer[0], layer[1]) for layer in list(zip(hidden_units[:-1],hidden_units[1:]))])
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        for linear in self.dnn_network:
            x = linear(x)
            x = F.relu(x)
        x = self.dropout(x)
        return x

class DeepFM(nn.Module):
    def __init__(self, feature_columns, hidden_units, dnn_dropout=0.):
        super(DeepFM, self).__init__()
        self.dense_feature_cols, self.sparse_feature_cols = feature_columns

        self.embed_layers = nn.ModuleDict({
            'embed_' + str(i):nn.Embedding(num_embeddings=feat['feat_num'],embedding_dim=feat['embed_dim'])
            for i, feat in enumerate(self.sparse_feature_cols)
        })

        self.fea_num = len(self.dense_feature_cols) + len(self.sparse_feature_cols) * self.sparse_feature_cols[0]['embed_dim']
        hidden_units.insert(0, self.fea_num)

        self.fm = FM(self.sparse_feature_cols[0]['embed_dim'], self.fea_num)
        self.dnn_network = Dnn(hidden_units, dnn_dropout)
        self.nn_final_linear = nn.Linear(hidden_units[-1], 1)

    def forward(self, x):
        dense_len = len(self.dense_feature_cols)
        dense_inputs, sparse_inputs = x[:, :dense_len], x[:, dense_len:]
        sparse_inputs = sparse_inputs.long()

        sparse_embeds = [self.embed_layers['embed_' + str(i)](sparse_inputs[:, i]) for i in range(sparse_inputs.shape[1])]
        sparse_embeds = torch.cat(sparse_embeds, dim=-1)

        x = torch.cat([sparse_embeds, dense_inputs], dim=-1)
        wide_outputs = self.fm(x)
        deep_outputs = self.nn_final_linear(self.dnn_network(x))

        outputs = F.sigmoid(torch.add(wide_outputs, deep_outputs))
        outputs = outputs.squeeze(-1)
        return outputs

hidden_units = [128,64, 32]

dnn_dropout = 0
model = DeepFM(fea_cols, hidden_units, dnn_dropout)
summary(model, input_shape=(trn_x.shape[1],))

for fea, label in iter(dl_train):
    out = model(fea)
    print(out)
    break


def auc(y_pred, y_true):
    pred = y_pred.data
    y = y_true.data
    return roc_auc_score(y, pred)

loss_func = nn.BCELoss()
optimizer = optim.Adam(params=model.parameters(), lr=0.001)
metric_func = auc
metric_name = 'auc'

%time
epochs = 100
log_step_freq = 10
dfhistory = pd.DataFrame(columns=['epoch', 'loss', metric_name, 'val_loss','val_' + metric_name])
print('start_training....')
nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('=========='*8 + '%s'%nowtime)

for epoch in range(1, epochs + 1):
    model.train()
    loss_sum = 0.0
    metric_sum  = 0.0
    step = 1

    for step, (features, labels) in enumerate(dl_train,1):
        optimizer.zero_grad()

        predictions = model(features)
        loss = loss_func(predictions, labels)
        try:
            metric = metric_func(predictions, labels)
        except ValueError:
            pass

        loss.backward()
        optimizer.step()

        loss_sum += loss.item()
        metric_sum += metric.item()
        if step % log_step_freq ==0:
            print(('[step=%d] loss:%.3f, ' + metric_name + ': %.3f') %(step, loss_sum/step, metric_sum/step))


    model.eval()
    val_loss_sum = 0.0
    val_metric_sum = 0.0
    val_step = 1

    for val_step, (features, labels) in enumerate(dl_val, 1):
        with torch.no_grad():
            predictions = model(features)
            val_loss = loss_func(predictions, labels)
            try:
                val_metric = metric_func(predictions, labels)
            except ValueError:
                pass

        val_loss_sum += val_loss.item()
        val_metric_sum += val_metric.item()

    info = (epoch, loss_sum/step, metric_sum/step, val_loss_sum/val_step, val_metric_sum/val_step)
    dfhistory.loc[epoch - 1] = info

    print(('\nEpoch=%d, loss=%.3f, ' + metric_name + ' =%.3f, val_loss=%.3f, ' + "val_" + metric_name + " =%.3f")%info)
    nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')   
    print('\n' + '===='*8 + '%s' %nowtime)
print('Finished Training')
dfhistory


def plot_metric(dfhistory, metric):
    train_metrics = dfhistory[metric]
    val_metrics = dfhistory['val_' + metric]
    epochs = range(1, len(train_metrics) + 1)
    plt.plot(epochs, train_metrics, 'bo--')
    plt.plot(epochs, val_metrics, 'ro-')
    plt.title('Training and validation' + metric)
    plt.xlabel('Epochs')
    plt.ylabel(metric)
    plt.legend(['train_' + metric, 'val_' + metric])
    plt.show()

plot_metric(dfhistory, 'loss')
plot_metric(dfhistory, 'auc')