gru_rec推荐诗

最新推荐文章于 2025-04-26 19:53:27 发布

东方佑

最新推荐文章于 2025-04-26 19:53:27 发布

阅读量126

点赞数

分类专栏： AIGC 量子变法文章标签：算法推荐算法机器学习数据结构 python

本文链接：https://blog.csdn.net/weixin_32759777/article/details/139212692

版权

量子变法同时被 2 个专栏收录

445 篇文章

订阅专栏

AIGC

40 篇文章

订阅专栏

代码


import paddle
from paddle import nn
from paddle.io import DataLoader, Dataset
import paddle.nn.functional as F
import pandas as pd
import numpy as np
import copy
import os
import math
import random
from sklearn.metrics import roc_auc_score,log_loss
from sklearn.preprocessing import normalize
from tqdm import tqdm
from collections import defaultdict
from sklearn.manifold import TSNE
from matplotlib import pyplot as plt
import faiss
# paddle.device.set_device('gpu:0')
import warnings
warnings.filterwarnings("ignore")

class SeqnenceDataset(Dataset):
    def __init__(self, config, df, voc,phase='train'):

        self.config = config
        self.max_length = self.config['max_length']
        self.user2item = df
        self.user_list = [voc.index(i) for i in voc if voc.index(i) in list(df.keys())]
        self.phase = phase

    def __len__(self, ):
        return len(self.user2item)

    def __getitem__(self, index):
        if self.phase == 'train':
            user_id = self.user_list[index]
            item_list = self.user2item[user_id]
            hist_item_list = []
            hist_mask_list = []

            k = random.choice(range(4, len(item_list)))  # 从[8,len(item_list))中随机选择一个index
            # k = np.random.randint(2,len(item_list))
            item_id = item_list[k]  # 该index对应的item加入item_id_list

            if k >= self.max_length:  # 选取seq_len个物品
                hist_item_list.append(item_list[k - self.max_length: k])
                hist_mask_list.append([1.0] * self.max_length)
            else:
                hist_item_list.append(item_list[:k] + [0] * (self.max_length - k))
                hist_mask_list.append([1.0] * k + [0.0] * (self.max_length - k))

            return paddle.to_tensor(hist_item_list).squeeze(0), paddle.to_tensor(hist_mask_list).squeeze(
                0), paddle.to_tensor([item_id])
        else:
            user_id = self.user_list[index]
            item_list = self.user2item[user_id]
            hist_item_list = []
            hist_mask_list = []

            k = int(0.8 * len(item_list))
            # k = len(item_list)-1

            if k >= self.max_length:  # 选取seq_len个物品
                hist_item_list.append(item_list[k - self.max_length: k])
                hist_mask_list.append([1.0] * self.max_length)
            else:
                hist_item_list.append(item_list[:k] + [0] * (self.max_length - k))
                hist_mask_list.append([1.0] * k + [0.0] * (self.max_length - k))

            return paddle.to_tensor(hist_item_list).squeeze(0), paddle.to_tensor(hist_mask_list).squeeze(
                0), item_list[k:]

    def get_test_gd(self):
        self.test_gd = {}
        for user in self.user2item:
            item_list = self.user2item[user]
            test_item_index = int(0.8 * len(item_list))
            self.test_gd[user] = item_list[test_item_index:]
        return self.test_gd
class GRU4Rec(nn.Layer):
    def __init__(self, config):
        super(GRU4Rec, self).__init__()

        self.config = config
        self.embedding_dim = self.config['embedding_dim']
        self.max_length = self.config['max_length']
        self.n_items = self.config['n_items']
        self.num_layers = self.config['num_layers']

        self.item_emb = nn.Embedding(self.n_items, self.embedding_dim, padding_idx=0)
        self.gru = nn.GRU(
            input_size=self.embedding_dim,
            hidden_size=self.embedding_dim,
            num_layers=self.num_layers,
            time_major=False,
        )
        self.loss_fun = nn.CrossEntropyLoss()
        self.reset_parameters()

    def calculate_loss(self,user_emb,pos_item):
        all_items = self.item_emb.weight
        scores = paddle.matmul(user_emb, all_items.transpose([1, 0]))
        return self.loss_fun(scores,pos_item)

    def output_items(self):
        return self.item_emb.weight

    def reset_parameters(self, initializer=None):
        for weight in self.parameters():
            paddle.nn.initializer.KaimingNormal(weight)

    def forward(self, item_seq, mask, item, train=True):
        seq_emb = self.item_emb(item_seq)
        seq_emb,_ = self.gru(seq_emb)
        user_emb = seq_emb[:,-1,:] #取GRU输出的最后一个Hidden作为User的Embedding
        if train:
            loss = self.calculate_loss(user_emb,item)
            output_dict = {
                'user_emb':user_emb,
                'loss':loss
            }
        else:
            output_dict = {
                'user_emb':user_emb
            }
        return output_dict

def my_collate(batch):
    hist_item, hist_mask, item_list = list(zip(*batch))

    hist_item = [x.unsqueeze(0) for x in hist_item]
    hist_mask = [x.unsqueeze(0) for x in hist_mask]

    hist_item = paddle.concat(hist_item, axis=0)
    hist_mask = paddle.concat(hist_mask, axis=0)
    return hist_item, hist_mask, item_list


def save_model(model, path):
    if not os.path.exists(path):
        os.makedirs(path)
    paddle.save(model.state_dict(), path + 'model.pdparams')


def load_model(model, path):
    state_dict = paddle.load(path + 'model.pdparams')
    model.set_state_dict(state_dict)
    print('model loaded from %s' % path)
    return model


def get_predict(model, test_data, hidden_size, topN=20):
    item_embs = model.output_items().cpu().detach().numpy()
    item_embs = normalize(item_embs, norm='l2')
    gpu_index = faiss.IndexFlatIP(hidden_size)
    gpu_index.add(item_embs)

    test_gd = dict()
    preds = dict()

    user_id = 0

    for (item_seq, mask, targets) in tqdm(test_data):

        # 获取用户嵌入
        # 多兴趣模型，shape=(batch_size, num_interest, embedding_dim)
        # 其他模型，shape=(batch_size, embedding_dim)
        user_embs = model(item_seq, mask, None, train=False)['user_emb']
        user_embs = user_embs.cpu().detach().numpy()

        # 用内积来近邻搜索，实际是内积的值越大，向量越近（越相似）
        if len(user_embs.shape) == 2:  # 非多兴趣模型评估
            user_embs = normalize(user_embs, norm='l2').astype('float32')
            D, I = gpu_index.search(user_embs, topN)  # Inner Product近邻搜索，D为distance，I是index
            #             D,I = faiss.knn(user_embs, item_embs, topN,metric=faiss.METRIC_INNER_PRODUCT)
            for i, iid_list in enumerate(targets):  # 每个用户的label列表，此处item_id为一个二维list，验证和测试是多label的
                test_gd[user_id] = iid_list
                preds[user_id] = I[i, :]
                user_id += 1
        else:  # 多兴趣模型评估
            ni = user_embs.shape[1]  # num_interest
            user_embs = np.reshape(user_embs,
                                   [-1, user_embs.shape[-1]])  # shape=(batch_size*num_interest, embedding_dim)
            user_embs = normalize(user_embs, norm='l2').astype('float32')
            D, I = gpu_index.search(user_embs, topN)  # Inner Product近邻搜索，D为distance，I是index
            #             D,I = faiss.knn(user_embs, item_embs, topN,metric=faiss.METRIC_INNER_PRODUCT)
            for i, iid_list in enumerate(targets):  # 每个用户的label列表，此处item_id为一个二维list，验证和测试是多label的
                recall = 0
                dcg = 0.0
                item_list_set = []

                # 将num_interest个兴趣向量的所有topN近邻物品（num_interest*topN个物品）集合起来按照距离重新排序
                item_list = list(
                    zip(np.reshape(I[i * ni:(i + 1) * ni], -1), np.reshape(D[i * ni:(i + 1) * ni], -1)))
                item_list.sort(key=lambda x: x[1], reverse=True)  # 降序排序，内积越大，向量越近
                for j in range(len(item_list)):  # 按距离由近到远遍历推荐物品列表，最后选出最近的topN个物品作为最终的推荐物品
                    if item_list[j][0] not in item_list_set and item_list[j][0] != 0:
                        item_list_set.append(item_list[j][0])
                        if len(item_list_set) >= topN:
                            break
                test_gd[user_id] = iid_list
                preds[user_id] = item_list_set
                user_id += 1
    return test_gd, preds


def evaluate(preds, test_gd, topN=50):
    total_recall = 0.0
    total_ndcg = 0.0
    total_hitrate = 0
    for user in test_gd.keys():
        recall = 0
        dcg = 0.0
        item_list = test_gd[user]
        for no, item_id in enumerate(item_list):
            if item_id in preds[user][:topN]:
                recall += 1
                dcg += 1.0 / math.log(no + 2, 2)
            idcg = 0.0
            for no in range(recall):
                idcg += 1.0 / math.log(no + 2, 2)
        total_recall += recall * 1.0 / len(item_list)
        if recall > 0:
            total_ndcg += dcg / idcg
            total_hitrate += 1
    total = len(test_gd)
    recall = total_recall / total
    ndcg = total_ndcg / total
    hitrate = total_hitrate * 1.0 / total
    return {f'recall@{topN}': recall, f'ndcg@{topN}': ndcg, f'hitrate@{topN}': hitrate}


def plot_embedding(data, title):
    x_min, x_max = np.min(data, 0), np.max(data, 0)
    data = (data - x_min) / (x_max - x_min)

    fig = plt.figure(dpi=120)
    plt.scatter(data[:, 0], data[:, 1], marker='.')

    plt.xticks([])
    plt.yticks([])
    plt.title(title)
    plt.show()
# 指标计算
def evaluate_model(model, test_loader, embedding_dim, topN=20):
    test_gd, preds = get_predict(model, test_loader, embedding_dim, topN=topN)
    return evaluate(preds, test_gd, topN=topN)
def train():
    config = {
        'train_path': 'train_set.pkl',
        'valid_path': 'val_set.pkl',
        'test_path': 'test_set.pkl',
        'voc_path': 'voc.pkl',
        'lr': 1e-4,
        'Epoch': 30,
        'batch_size': 512,
        'embedding_dim': 64,
        'max_length': 100,
        'n_items': 6912,
        'K': 4,
        'num_layers':3
    }


    # 读取数据
    train_df = pd.read_pickle(config['train_path'])
    valid_df = pd.read_pickle(config['valid_path'])
    test_df = pd.read_pickle(config['test_path'])
    voc = pd.read_pickle(config['voc_path'])
    train_dataset = SeqnenceDataset(config, train_df, voc, phase='train')
    valid_dataset = SeqnenceDataset(config, valid_df, voc, phase='test')
    test_dataset = SeqnenceDataset(config, test_df, voc, phase='test')
    train_loader = DataLoader(dataset=train_dataset, batch_size=config['batch_size'], shuffle=True, num_workers=8)
    valid_loader = DataLoader(dataset=valid_dataset, batch_size=config['batch_size'], shuffle=False,
                              collate_fn=my_collate)
    test_loader = DataLoader(dataset=test_dataset, batch_size=config['batch_size'], shuffle=False,
                             collate_fn=my_collate)

    model = GRU4Rec(config)
    optimizer = paddle.optimizer.Adam(parameters=model.parameters(), learning_rate=config['lr'])
    log_df = pd.DataFrame()
    best_reacall = -1

    exp_path = './exp/ml-20m_softmax/MIND_{}_{}_{}/'.format(config['lr'], config['batch_size'], config['embedding_dim'])
    os.makedirs(exp_path, exist_ok=True, mode=0o777)
    patience = 5
    last_improve_epoch = 1
    log_csv = exp_path + 'log.csv'
    # *****************************************************train*********************************************
    for epoch in range(1, 1 + config['Epoch']):
        # try :
        pbar = tqdm(train_loader)
        model.train()
        loss_list = []

        print()
        print('Training:')
        print()
        for batch_data in pbar:
            (item_seq, mask, item) = batch_data

            output_dict = model(item_seq, mask, item)
            loss = output_dict['loss']

            loss.backward()
            optimizer.step()
            optimizer.clear_grad()

            loss_list.append(loss.item())

            pbar.set_description('Epoch [{}/{}]'.format(epoch, config['Epoch']))
            pbar.set_postfix(loss=np.mean(loss_list))
        # *****************************************************valid*********************************************

        print('Valid')
        recall_metric = evaluate_model(model, valid_loader, config['embedding_dim'], topN=50)
        print(recall_metric)
        recall_metric['phase'] = 'valid'
        if log_df.empty:
            log_df = pd.DataFrame({k: [v] for k, v in recall_metric.items()})
        else:

            log_df = pd.concat([pd.DataFrame({k: [v] for k, v in recall_metric.items()}), log_df])
        log_df.to_csv(log_csv)

        if recall_metric['recall@50'] > best_reacall:
            save_model(model, exp_path)
            best_reacall = recall_metric['recall@50']
            last_improve_epoch = epoch

        if epoch - last_improve_epoch > patience:
            break

    print('Testing')
    model = load_model(model, exp_path)
    recall_metric = evaluate_model(model, test_loader, config['embedding_dim'], topN=50)
    print(recall_metric)
    recall_metric['phase'] = 'test'

    log_df = pd.concat([pd.DataFrame({k: [v] for k, v in recall_metric.items()}), log_df])
    log_df.to_csv(log_csv)
    item_emb = model.output_items().numpy()
    tsne_emb = TSNE(n_components=2).fit_transform(item_emb)
    plot_embedding(tsne_emb, 'GRU4Rec Item Embedding')
if __name__ == '__main__':
    train()