flink 实现 dau 去重(状态实现)
object DauFirstInfo{
def main(args: Array[String]):Unit = {
val testDauGroupId = "test_daufirstinfo_20240303"
val source_kafka_host = ConfigHelper.getBootStrapServers(Config.TEST_SOURCE_KAFKA,Region.TEST_CLUSTER.getName())
val testTopicProprties:Properties = KafkaConnector.createConsumerProperties(source_kafka_host,testDauGroupId)
val testConsumer = KafkaConnector.createStringSource(Config.TEST_SOURCE_KAFKA,testTopicProprties)
val env:StreamExecutionEnvironment = StreamingUtils.getStreamExecutionment
env.getCheckpointConfig.setMaxConcurrentCheckpoints(1)
val dataStream = env
.addSource(testConsumer).uid("xxx").name("xxx")
.flatMap(new RichFlatMapFunction[String,String]){
override def flatMap(in:String,collector:Collector[String]): Unit = {
val sourceObject: JsonObject = new JsonParser().parse(in).getJsonObject()
if(StringUnits.isNotBlank(sourceObject.get("shebei_id").getAsString) && sourceObject.get("qidong_timestamp").getAsLong){
collector.collect(in)
}
}
}).uid("map_valid_info").name("map_valid_info")
.keyBy(new KeySelector[String,String]{
override def getKey(value:String):String = {
val sourceObject: JsonObject = new JsonParser().parse(in).getJsonObject()
sourceObject.get("sheibei_id").getAsString + DateFormatUtils.format(sourceObject.get("qidong_timestamp").getAsLong,"yyyy-MM-dd")
}
})
.flatMap(new FirstNotBlankDauProcess()).uid("first_dau_state").name("first_dau_state")
.setMaxParallelism(100)
val sink_kafka_host:String = Confighelper.getBootstrapServers(CommonConfig.TEST_SINK_KAFKA,Region.TEST_CLUSTER.getName())
val props: Properties = new Properties()
props.put("bootstrap.servers",sink_kafka_host)
props.put("request.timeout.ms",200000)
val kafkaProducer: FlinkKafkaProducer[String] = kafkaConnector.createStringSink(Commonfig.TEST_SINK_KAFKA,props)
dataStream.addSink(kafkaProducer).uid("sink_id").name("sink_name").setParallelism(30)
env.execute("ValidDauJob")
}
}
class FirstNotBlankDauProcess extends RichFlatMapFunction[String,String]{
var clientState : ValueState[Integer] = _
override def open(parameters: Configuration): Unit = {
super.open(parameters)
val stateTtlConfig: StateTtlConfig = StateTtlConfig
.newBulider(Time.minutes(1560))
.useProcessingTime()
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
.setStateVisibility(StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp)
.bulid
val roaringBitmapDescriptor = new ValueStateDescriptor[Integer]("FirstNotBlankDauState",Types.GENERIC(classOf[Integer]))
roaringBitmapDescriptor.enableTimeToLive(stateTtlConfig)
clientState = getRuntimeContext.getSate[Integer](roaringBitmapDescriptor)
}
override def flatMap(in:String,out:Collector[String]): Unit = {
val existFlag = clientState.value()
if(existFlag != 1){
clientstate.update(1)
out.collect(in)
}
}
override def close(): Unit = {
super.close()
}
}