1.这是一个普通的flink流处理任务,读取kafka的数据,然后输出。
package com.flink.java.project;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import org.apache.flink.util.Collector;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import java.util.Iterator;
import java.util.Properties;
public class FlinkUserJob {
public static void main(String[] args) throws Exception {
// 获取env
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//设置并行度
env.setParallelism(3);
//checkpoint的周期
// env.enableCheckpointing(10000);
// //设置模式为:exactly_one,仅一次语义
// env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
// //checkpoint最小间隔
// env.getCheckpointConfig().setMinPauseBetweenCheckpoints(10000);
// //检查点必须在1分钟之内完成,或者被丢弃【checkpoint超时时间】
// env.getCheckpointConfig().setCheckpointTimeout(10000);
// //同一时间只允许进行一次检查点
// env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
// //表示一旦Flink程序被cancel后,会保留checkpoint数据
// env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
// // 设置state backend
// env.setStateBackend(new FsStateBackend("file:///D:\\YYYY"));
//设置kafka消费参数
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.10.15:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, "flink-k2k_01");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");;
FlinkKafkaConsumer011<String> kafkaConsumer011 = new FlinkKafkaConsumer011<String>("user_data",new SimpleStringSchema(), props);
kafkaConsumer011.setStartFromLatest();
//加入kafka数据源
DataStreamSource<String> userSource = env.addSource(kafkaConsumer011).setParallelism(1);
userSource.map(new MapFunction<String, Tuple5<String,String,String,String,String>>() {
@Override
public Tuple5<String,String,String,String,String> map(String str) throws Exception {
String[] split = str.split(",");
return new Tuple5<>(split[0],split[1],split[2],split[3],split[4]);
}
}).keyBy(0)
.countWindow(3)
.apply(new WindowFunction<Tuple5<String,String, String,String,String>, Object, Tuple, GlobalWindow>() {
@Override
public void apply( Tuple tuple, GlobalWindow window, Iterable<Tuple5<String, String, String,String, String>> input, Collector<Object> out) {
Iterator<Tuple5<String, String,String,String,String>> iterator = input.iterator();
UserInfo user=new UserInfo();
while (iterator.hasNext()) {
Tuple5<String,String,String, String,String> tp = iterator.next();
switch (tp.f4){
//用户id uname birthday sex f1
case "f1": {
user.setUserID(tp.f0);
user.setName(tp.f1);
user.setAge(tp.f2);
user.setSex(tp.f3);
};
//uid,categoryid,productid,time,f2
case "f2": {
user.setPro(tp.f3);
};
//userid productid amount time f3
case "f3": {
user.setMoney(tp.f2);
user.setTime(tp.f3);
};
}
}
out.collect( user.toString());
}
}).print();
env.execute(FlinkUserJob.class.getName());
}
}
2.完善代码后加入状态的管理,如下
package com.flink.java.project;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.state.StateTtlConfig;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import org.apache.flink.util.Collector;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import java.util.Properties;
/**
* @Author BaronND
* @Description 消费kafka消息,
* 如果数据量大的话,加上状态管理
*
*/
public class FlinkUserJob_v2 {
public static void main(String[] args) throws Exception {
// 获取env
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//设置并行度
env.setParallelism(3);
//checkpoint的周期
env.enableCheckpointing(10000);
//设置模式为:exactly_one,仅一次语义
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
// //checkpoint最小间隔
// env.getCheckpointConfig().setMinPauseBetweenCheckpoints(10000);
// //检查点必须在1分钟之内完成,或者被丢弃【checkpoint超时时间】
// env.getCheckpointConfig().setCheckpointTimeout(10000);
// //同一时间只允许进行一次检查点
// env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
// //表示一旦Flink程序被cancel后,会保留checkpoint数据
// env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
// 设置state backend
env.setStateBackend(new FsStateBackend("file:///D:\\tmp\\flinkStateBackend"));
//设置kafka消费参数
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.10.15:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, "flink-k2k_01");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");;
FlinkKafkaConsumer011<String> kafkaConsumer011 = new FlinkKafkaConsumer011<String>("user_data",new SimpleStringSchema(), props);
kafkaConsumer011.setStartFromLatest();
//加入kafka数据源
DataStreamSource<String> userSource = env.addSource(kafkaConsumer011).setParallelism(1);
userSource.map(new MapFunction<String, Tuple5<String,String,String,String,String>>() {
@Override
public Tuple5<String,String,String,String,String> map(String str) throws Exception {
String[] split = str.split(",");
return new Tuple5<>(split[0],split[1],split[2],split[3],split[4]);
}
}).keyBy(0)
.flatMap(new CountWindowPrint())
.print();
env.execute(FlinkUserJob.class.getName());
}
}
class CountWindowPrint extends RichFlatMapFunction<Tuple5<String, String, String,String, String>, UserInfo> {
private static final long serialVersionUID = 1808329479322205953L;
/**
* //用户id,名字,性别,年龄分段,产品,金额,下单时间
*/
private transient ValueState<Tuple2<Integer,UserInfo>> data;
StateTtlConfig ttlConfig =
StateTtlConfig.newBuilder(Time.seconds(1)) //它是生存时间值
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
//状态可见性配置是否在读取访问时返回过期值
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
.build();
@Override
public void flatMap(Tuple5<String, String, String,String, String> input, Collector<UserInfo> out) throws Exception {
//用户id,名字,性别,年龄分段,产品,金额,下单时间
Tuple2<Integer, UserInfo> tuple2 = data.value();
// update the count
int count=tuple2.f0 +1;
Tuple5<String,String,String, String,String> tp = input;
final UserInfo user = tuple2.f1;
switch (tp.f4){
//用户id uname birthday sex f1
case "f1": {
user.setUserID(tp.f0);
user.setName(tp.f1);
user.setAge(tp.f2);
user.setSex(tp.f3);
};
//uid,categoryid,productid,time,f2
case "f2": {
user.setPro(tp.f3);
};
//userid productid amount time f3
case "f3": {
user.setMoney(tp.f2);
user.setTime(tp.f3);
};
}
data.update(Tuple2.of(count,user));
// if the count reaches 2, emit the average and clear the state
if (count >= 3) {
out.collect(user);//sout
data.clear();
}
}
@Override
public void open(Configuration config) {
ValueStateDescriptor<Tuple2<Integer,UserInfo>> descriptor =
new ValueStateDescriptor<>(
"userInfo", // the state name
TypeInformation.of(new TypeHint<Tuple2<Integer,UserInfo>>() {}), // type information
new Tuple2<>(0,new UserInfo())); // default value of the state, if nothing was set
//设置stage过期时间
//descriptor.enableTimeToLive(ttlConfig);
data = getRuntimeContext().getState(descriptor);
}
}