redis中存储了部分域名与ip的对应关系数据,由于网络或者其他原因,会导致域名的ip地址为8.8.8.8,所以需要将这部分数据更新,现用flink进行相关操作。
package com.myproject.analyse
import com.myproject.utils.{RedisUtil, UrlUtil}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
import org.apache.flink.streaming.api.functions.source.{RichSourceFunction, SourceFunction}
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper}
import redis.clients.jedis.Jedis
import scala.collection.JavaConverters._
/**
* Author: Admin
* Date: 2023/3/15 9:54
* Description:
* 将redis中8.8.8.8的ip重新获取(后续可将filter去掉 全量重新获取的和原有的不同时 更新)
**/
object SourceAndSinkRedis {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
// env.setParallelism(4)
/* val conf = new FlinkJedisPoolConfig.Builder()
.setHost("192.168.0.30")
.setDatabase(2)
.build()*/
val stream = env.addSource(new MyRedisSource("*")).filter(_._2 == "8.8.8.8")
// stream.print()
stream.map(x => {
val (host, oldIp) = x
val ip = UrlUtil.getHostIp(host)
((host, ip), oldIp)
}).filter(x => {
x._1._2 != x._2
}).map(_._1)
// .addSink(new RedisSink[(String, String)](conf, new MyRedisSink))
.addSink(new MyRedisSinkSet)
env.execute()
}
class MyRedisSource(keyPattern: String) extends RichSourceFunction[(String, String)] {
var jedisClient: Jedis = _
override def open(parameters: Configuration): Unit = {
jedisClient = RedisUtil.getJedisClient
jedisClient.select(2)
}
override def run(ctx: SourceFunction.SourceContext[(String, String)]): Unit = {
val keys = jedisClient.keys(keyPattern)
/* val keysIter = keys.iterator()
while (keysIter.hasNext) {
val key = keysIter.next()
ctx.collect(key, jedisClient.get(key))
}*/
keys.asScala.foreach(key => {
ctx.collect(key, jedisClient.get(key))
})
}
override def cancel(): Unit = {
jedisClient.close()
}
}
class MyRedisSink extends RedisMapper[(String, String)] {
override def getCommandDescription: RedisCommandDescription = new RedisCommandDescription(RedisCommand.SET)
override def getKeyFromData(data: (String, String)): String = data._1
override def getValueFromData(data: (String, String)): String = data._2
}
class MyRedisSinkSet extends RichSinkFunction[(String, String)] {
var jedisClient: Jedis = _
override def open(parameters: Configuration): Unit = {
jedisClient = RedisUtil.getJedisClient
jedisClient.select(2)
}
override def invoke(value: (String, String)): Unit = {
println(value)
if (value._2 != null)
jedisClient.set(value._1, value._2)
}
override def close(): Unit = jedisClient.close()
}
}