操作的数据
链接:https://pan.baidu.com/s/1pXyWDoIf0-QbZbtUPDQxRw
提取码:yyds
--来自百度网盘超级会员V4的分享
创建对应数据的类UserBehavior
public class UserBehavior {
public String userId;
public String productId;
public String categoryId;
public String type;
public Long ts;
public UserBehavior() {
}
public UserBehavior(String userId, String productId, String categoryId, String type, Long ts) {
this.userId = userId;
this.productId = productId;
this.categoryId = categoryId;
this.type = type;
this.ts = ts;
}
@Override
public String toString() {
return "UserBehavior{" +
"userId='" + userId + '\'' +
", productId='" + productId + '\'' +
", categoryId='" + categoryId + '\'' +
", type='" + type + '\'' +
", ts=" + new Timestamp(ts) +
'}';
}
}
创建统计的类ProductViewCountPerWindow
public class ProductViewCountPerWindow {
public String productId;
public Long count;
public Long windowStartTime;
public Long windowEndTime;
public ProductViewCountPerWindow() {
}
public ProductViewCountPerWindow(String productId, Long count, Long windowStartTime, Long windowEndTime) {
this.productId = productId;
this.count = count;
this.windowStartTime = windowStartTime;
this.windowEndTime = windowEndTime;
}
@Override
public String toString() {
return "(" +
"商品Id=" + productId +
", 浏览次数=" + count +
", " + new Timestamp(windowStartTime) +
"~" + new Timestamp(windowEndTime) +
")";
}
}
实例代码-flink实时每五分钟查看前面一个小时的数据的商品访问量排名
public class Example1 {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env
.readTextFile("C:\\Users\\zhang\\Desktop\\flink\\src\\main\\resources\\UserBehavior.csv")
.map(new MapFunction<String, UserBehavior>() {
@Override
public UserBehavior map(String in) throws Exception {
String[] array = in.split(",");
return new UserBehavior(
array[0], array[1], array[2], array[3],
Long.parseLong(array[4]) * 1000L
);
}
})
.filter(r -> r.type.equals("pv"))
.assignTimestampsAndWatermarks(
WatermarkStrategy.<UserBehavior>forBoundedOutOfOrderness(Duration.ofSeconds(5))
.withTimestampAssigner(new SerializableTimestampAssigner<UserBehavior>() {
@Override
public long extractTimestamp(UserBehavior element, long recordTimestamp) {
return element.ts;
}
})
)
.keyBy(r -> r.productId)
.window(SlidingEventTimeWindows.of(Time.hours(1), Time.minutes(5)))
.aggregate(
new AggregateFunction<UserBehavior, Long, Long>() {
@Override
public Long createAccumulator() {
return 0L;
}
@Override
public Long add(UserBehavior in, Long accumulator) {
return accumulator + 1;
}
@Override
public Long getResult(Long accumulator) {
return accumulator;
}
@Override
public Long merge(Long a, Long b) {
return null;
}
},
new ProcessWindowFunction<Long, ProductViewCountPerWindow, String, TimeWindow>() {
@Override
public void process(String key, Context ctx, Iterable<Long> elements, Collector<ProductViewCountPerWindow> out) throws Exception {
out.collect(new ProductViewCountPerWindow(
key,
elements.iterator().next(),
ctx.window().getStart(),
ctx.window().getEnd()
));
}
}
)
.keyBy(r -> r.windowEndTime)
.process(new TopN(3))
.print();
env.execute();
}
public static class TopN extends KeyedProcessFunction<Long, ProductViewCountPerWindow, String> {
private int n; // 前n名
public TopN(int n) {
this.n = n;
}
private ListState<ProductViewCountPerWindow> listState;
@Override
public void open(Configuration parameters) throws Exception {
listState = getRuntimeContext().getListState(
new ListStateDescriptor<ProductViewCountPerWindow>(
"list-state",
Types.POJO(ProductViewCountPerWindow.class)
)
);
}
@Override
public void processElement(ProductViewCountPerWindow in, Context ctx, Collector<String> out) throws Exception {
listState.add(in);
// 加1000ms是为了保证所有的ProductViewCountPerWindow全部到达
ctx.timerService().registerEventTimeTimer(
in.windowEndTime + 1000L
);
}
@Override
public void onTimer(long timerTs, OnTimerContext ctx, Collector<String> out) throws Exception {
// 将ListState中的数据取出并放入ArrayList
ArrayList<ProductViewCountPerWindow> arrayList = new ArrayList<>();
for (ProductViewCountPerWindow e : listState.get()) arrayList.add(e);
// 由于listState中的数据已经没用了,所以清空
listState.clear();
// 按照浏览次数降序排列
arrayList.sort(new Comparator<ProductViewCountPerWindow>() {
@Override
public int compare(ProductViewCountPerWindow p1, ProductViewCountPerWindow p2) {
return (int)(p2.count - p1.count);
}
});
StringBuilder result = new StringBuilder();
result.append("=========================================\n");
result.append("窗口结束时间:" + new Timestamp(timerTs - 1000L) + "\n");
for (int i = 0; i < n; i++) {
ProductViewCountPerWindow tmp = arrayList.get(i);
result.append("第" + (i+1) + "名的商品ID是:" + tmp.productId + ",浏览次数是:" + tmp.count + "\n");
}
result.append("=========================================\n");
out.collect(result.toString());
}
}
}