flink-操作redis

redis中存储了部分域名与ip的对应关系数据，由于网络或者其他原因，会导致域名的ip地址为8.8.8.8，所以需要将这部分数据更新，现用flink进行相关操作。

package com.myproject.analyse

import com.myproject.utils.{RedisUtil, UrlUtil}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
import org.apache.flink.streaming.api.functions.source.{RichSourceFunction, SourceFunction}
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper}
import redis.clients.jedis.Jedis
import scala.collection.JavaConverters._

/**
  * Author: Admin
  * Date: 2023/3/15 9:54
  * Description:
  * 将redis中8.8.8.8的ip重新获取（后续可将filter去掉 全量重新获取的和原有的不同时 更新）
  **/
object SourceAndSinkRedis {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    //    env.setParallelism(4)
    /* val conf = new FlinkJedisPoolConfig.Builder()
       .setHost("192.168.0.30")
       .setDatabase(2)
       .build()*/
    val stream = env.addSource(new MyRedisSource("*")).filter(_._2 == "8.8.8.8")

    //    stream.print()
    stream.map(x => {
      val (host, oldIp) = x
      val ip = UrlUtil.getHostIp(host)
      ((host, ip), oldIp)
    }).filter(x => {
      x._1._2 != x._2
    }).map(_._1)
      //      .addSink(new RedisSink[(String, String)](conf, new MyRedisSink))
      .addSink(new MyRedisSinkSet)


    env.execute()
  }

  class MyRedisSource(keyPattern: String) extends RichSourceFunction[(String, String)] {
    var jedisClient: Jedis = _

    override def open(parameters: Configuration): Unit = {
      jedisClient = RedisUtil.getJedisClient
      jedisClient.select(2)
    }

    override def run(ctx: SourceFunction.SourceContext[(String, String)]): Unit = {
      val keys = jedisClient.keys(keyPattern)
      /*  val keysIter = keys.iterator()
        while (keysIter.hasNext) {
          val key = keysIter.next()
          ctx.collect(key, jedisClient.get(key))
        }*/
      keys.asScala.foreach(key => {
        ctx.collect(key, jedisClient.get(key))
      })
    }

    override def cancel(): Unit = {
      jedisClient.close()
    }
  }


  class MyRedisSink extends RedisMapper[(String, String)] {
    override def getCommandDescription: RedisCommandDescription = new RedisCommandDescription(RedisCommand.SET)

    override def getKeyFromData(data: (String, String)): String = data._1

    override def getValueFromData(data: (String, String)): String = data._2
  }

  class MyRedisSinkSet extends RichSinkFunction[(String, String)] {
    var jedisClient: Jedis = _

    override def open(parameters: Configuration): Unit = {
      jedisClient = RedisUtil.getJedisClient
      jedisClient.select(2)
    }

    override def invoke(value: (String, String)): Unit = {
      println(value)
      if (value._2 != null)
        jedisClient.set(value._1, value._2)

    }

    override def close(): Unit = jedisClient.close()

  }

}