接口_kafka_UserTransmit
springboot接口
com.test.MainApplication
package com.test;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class MainApplication {
public static void main(String[] args) {
SpringApplication.run(MainApplication.class,args);
}
}
com.test.controller.AvroTest
package com.test.controller;
import com.test.util.kafka_utils.KafkaUtils;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.springframework.web.bind.annotation.*;
@RestController
public class AvroTest {
@PostMapping("/send)
public String getAvroInfo(@RequestBody String avroInfo){
System.out.println(avroInfo);
Producer producer = KafkaUtils.getKafkaProducer();
producer.send(new ProducerRecord<String, String>("user_message", avroInfo));
return "接收post请求的数据
}
}
UserTransmit
package com.test.app;
import com.test.bean.User;
import com.test.util.hdfsSink.HdfsUtils;
import com.test.util.kafka_utils.KafkaUtils;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
import java.time.Duration;
public class UserTransmit {
public static void main(String[] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.setParallelism(1);
SingleOutputStreamOperator userDS = env.addSource(KafkaUtils.getFlinkKafkaConsumer("user_message"))
.map(new MapFunction<String, User>() {
@Override
public User map(String s) throws Exception {
String[] split = s.split(",");
return new User(split[0], split[1], Long.valueOf(split[2])*1L);
}
});
//filter
// SingleOutputStreamOperator user_water_ds = userDS.assignTimestampsAndWatermarks(
// WatermarkStrategy.
// <User>forBoundedOutOfOrderness(Duration.ofSeconds(10))
// .withTimestampAssigner((data, ts) -> data.getTransmit_time() * 1000L)
// );
SingleOutputStreamOperator<Tuple2<String,User>> tuple_user_ds = userDS.map(new MapFunction<User, Tuple2<String, User>>() {
@Override
public Tuple2<String, User> map(User user) throws Exception {
return Tuple2.of((user.getUserid()), user);
}
});
SingleOutputStreamOperator<Tuple2<String, Long>> resultDS = tuple_user_ds
.keyBy(value -> value.f0)
.timeWindow(Time.days(1))
.process(new ProcessWindowFunction<Tuple2<String, User>, Tuple2<String, Long>, String, TimeWindow>() {
Long sumTransmit = 0L;
@Override
public void process(String s, Context context, Iterable<Tuple2<String, User>> elements, Collector<Tuple2<String, Long>> out) throws Exception {
sumTransmit++;
out.collect(Tuple2.of(s, sumTransmit));
// System.out.println(Tuple2.of(s, sumTransmit));
}
});
resultDS.addSink(new HdfsUtils<User>().getTBucketingSink("hdfs://192.168.1.162:8020/UserData/"));
try {
env.execute();
} catch (Exception e) {
e.printStackTrace();
}
}
}