项目架构
项目过程中的数据转换
注意:图中RDD后跟的不是元素类型,而是其中元素的含义,如:userId即为用户Id,score和rating均为用户对商品的打分
离线阶段数据转化流程
实时阶段数据转化流程
部分代码
商品推荐优先级计算(实时推荐阶段)
def computeProductScores(
simProducts:scala.collection.Map[Int,scala.collection.immutable.Map[Int,Doub
le]],userRecentlyRatings:Array[(Int,Double)],topSimProducts: Array[Int]):
Array[(Int,Double)] ={
//用于保存每一个待选商品和最近评分的每一个商品的权重得分
val score = scala.collection.mutable.ArrayBuffer[(Int,Double)]()
//用于保存每一个商品的增强因子数
val increMap = scala.collection.mutable.HashMap[Int,Int]()
//用于保存每一个商品的减弱因子数
val decreMap = scala.collection.mutable.HashMap[Int,Int]()
for (topSimProduct <- topSimProducts; userRecentlyRating <- userRecentlyRatings){
val simScore = getProductsSimScore(simProducts,userRecentlyRating._1,topSimProduct)
if(simScore > 0.6){
score += ((topSimProduct, simScore * userRecentlyRating._2 ))
if(userRecentlyRating._2 > 3){
increMap(topSimProduct) = increMap.getOrDefault(topSimProduct,0) + 1
}else{
decreMap(topSimProduct) = decreMap.getOrDefault(topSimProduct,0) + 1
}
}
}
score.groupBy(_._1).map{case (productId,sims) =>
(productId,sims.map(_._2).sum / sims.length + log(increMap.getOrDefault(productId, 1)) - log(decreMap.getOrDefault(productId, 1)))
}.toArray.sortWith(_._2>_._2)
}
自定义拦截器
package flume.interceptor;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
public class LogETLInterceptor implements Interceptor {
@Override
public void initialize() {
}
@Override
public Event intercept(Event event) {
// 1 获取数据
byte[] body = event.getBody();
String log = new String(body, Charset.forName("UTF-8"));
if (log.contains("PRODUCT_RATING_PREFIX:")) {
newBody = log.split(":")[1].trim()
event.setBody(newBody.getBytes())
return event;
}else {
return null;
}
}
@Override
public List<Event> intercept(List<Event> events) {
ArrayList<Event> interceptors = new ArrayList<>();
for (Event event : events) {
Event intercept1 = intercept(event);
if (intercept1 != null){
interceptors.add(intercept1);
}
}
return interceptors;
}
@Override
public void close() {
}
public static class Builder implements Interceptor.Builder{
@Override
public Interceptor build() {
return new LogETLInterceptor();
}
@Override
public void configure(Context context) {
}
}
}