有许多经典的推荐方案,值得学习
import pandas as pd
import numpy as np
from pandas.io import json
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
from ast import literal_eval
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from surprise import Reader, Dataset, SVD
from surprise.model_selection import cross_validate
# https://www.kaggle.com/code/ibtesama/getting-started-with-a-movie-recommendation-system
df1 = pd.read_csv('D:\\python\\dataset\\电影数据分析\\archive\\tmdb_5000_credits.csv')
df2 = pd.read_csv('D:\\python\\dataset\\电影数据分析\\archive\\tmdb_5000_movies.csv')
df1.columns = ['id', 'tittle', 'cast', 'crew']
df2 = df2.merge(df1, on='id')
# print(df2.head(10))
#
# C= df2['vote_average'].mean()
# print(C)
#
# m= df2['vote_count'].quantile(0.9)
# print(m)
#
# q_movies = df2.copy().loc[df2['vote_count'] >= m]
# print(q_movies.shape)
#
# def weighted_rating(x, m=m, C=C):
# v = x['vote_count']
# R = x['vote_average']
# # Calculation based on the IMDB formula
# return (v/(v+m) * R) + (m/(m+v) * C)
#
# q_movies['score'] = q_movies.apply(weighted_rating, axis=1)
# q_movies = q_movies.sort_values(by=['score'],ascending=False)
# good = q_movies.head(10)['title']
# print(good.tolist())
# print(good)
# pop = df2.sort_values('popularity', ascending=False)
# plt.figure(figsize=(12, 4))
#
# plt.barh(pop['title'].head(6), pop['popularity'].head(6), align='center',
# color='skyblue')
# # y轴逆序
# plt.gca().invert_yaxis()
# plt.xlabel("Popularity")
# plt.title("Popular Movies")
# plt.show()
ov = df2['overview'].head(5)
print(ov)
# Define a TF-IDF Vectorizer Object. Remove all english stop words such as 'the', 'a'
tfidf = TfidfVectorizer(stop_words='english')
# Replace NaN with an empty string
df2['overview'] = df2['overview'].fillna('')
# Construct the required TF-IDF matrix by fitting and transforming the data
tfidf_matrix = tfidf.fit_transform(df2['overview'])
# Output the shape of tfidf_matrix
print(tfidf_matrix.shape)
# Compute the cosine similarity matrix
cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)
# Construct a reverse map of indices and movie titles
indices = pd.Series(df2.index, index=df2['title']).drop_duplicates()
# Function that takes in movie title as input and outputs most similar movies
def get_recommendations(title, cosine_sim=cosine_sim):
# Get the index of the movie that matches the title
idx = indices[title]
# Get the pairwsie similarity scores of all movies with that movie
sim_scores = list(enumerate(cosine_sim[idx]))
# Sort the movies based on the similarity scores
sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
# Get the scores of the 10 most similar movies
sim_scores = sim_scores[1:11]
# Get the movie indices
movie_indices = [i[0] for i in sim_scores]
# Return the top 10 most similar movies
return df2['title'].iloc[movie_indices]
ts = get_recommendations('The Dark Knight Rises')
ta = get_recommendations('The Avengers')
features = ['cast', 'crew', 'keywords', 'genres']
for feature in features:
df2[feature] = df2[feature].apply(literal_eval)
# Get the director's name from the crew feature. If director is not listed, return NaN
def get_director(x):
for i in x:
if i['job'] == 'Director':
return i['name']
return np.nan
# Returns the list top 3 elements or entire list; whichever is more.
def get_list(x):
if isinstance(x, list):
names = [i['name'] for i in x]
# Check if more than 3 elements exist. If yes, return only first three. If no, return entire list.
if len(names) > 3:
names = names[:3]
return names
# Return empty list in case of missing/malformed data
return []
# Define new director, cast, genres and keywords features that are in a suitable form.
df2['director'] = df2['crew'].apply(get_director)
features = ['cast', 'keywords', 'genres']
for feature in features:
df2[feature] = df2[feature].apply(get_list)
# Print the new features of the first 3 films
h3 = df2[['title', 'cast', 'director', 'keywords', 'genres']].head(3)
print(h3)
# Function to convert all strings to lower case and strip names of spaces
def clean_data(x):
if isinstance(x, list):
return [str.lower(i.replace(" ", "")) for i in x]
else:
# Check if director exists. If not, return empty string
if isinstance(x, str):
return str.lower(x.replace(" ", ""))
else:
return ''
# Apply clean_data function to your features.
features = ['cast', 'keywords', 'director', 'genres']
for feature in features:
df2[feature] = df2[feature].apply(clean_data)
def create_soup(x):
return ' '.join(x['keywords']) + ' ' + ' '.join(x['cast']) + ' ' + x['director'] + ' ' + ' '.join(x['genres'])
df2['soup'] = df2.apply(create_soup, axis=1)
count = CountVectorizer(stop_words='english')
count_matrix = count.fit_transform(df2['soup'])
print(count_matrix)
# Compute the Cosine Similarity matrix based on the count_matrix
cosine_sim2 = cosine_similarity(count_matrix, count_matrix)
# Reset index of our main DataFrame and construct reverse mapping as before
df2 = df2.reset_index()
indices = pd.Series(df2.index, index=df2['title'])
re2 = get_recommendations('The Dark Knight Rises', cosine_sim2)
print(re2)
re3 = get_recommendations('The Godfather', cosine_sim2)
print(re3)
reader = Reader()
ratings = pd.read_csv('D:\\python\\dataset\\电影数据分析\\archive\\ratings_small.csv')
ratings.head()
data = Dataset.load_from_df(ratings[['userId', 'movieId', 'rating']], reader)
#data.split(n_folds=5)
svd = SVD()
# 新版本已经弃用 evaluate 改用cross_validate
cv = cross_validate(algo=svd,data=data,measures=['RMSE', 'MAE'],cv=5,verbose=True)
print(cv)
trainSet = data.build_full_trainset()
svd.fit(trainSet)
userId1 = ratings[ratings['userId'] == 1]
print(userId1)
result = svd.predict(1, 302, 3)
print(result)