Flink生产---15Redis、JedisUtil

Flink 项目中访问 Redis 的方法都是自己进行的实现，推荐使用 Bahir 连接器。

在本地单机情况下：

public static class RedisExampleMapper implements RedisMapper<Tuple2<String, String>>{
    @Override
    public RedisCommandDescription getCommandDescription() {
        return new RedisCommandDescription(RedisCommand.HSET, "HASH_NAME");
    }
    @Override
    public String getKeyFromData(Tuple2<String, String> data) {
        return data.f0;
    }
    @Override
    public String getValueFromData(Tuple2<String, String> data) {
        return data.f1;
    }
}
FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder().setHost("127.0.0.1").build();
DataStream<String> stream = ...;
stream.addSink(new RedisSink<Tuple2<String, String>>(conf, new RedisExampleMapper());

当然我们也可以使用在集群或者哨兵模式下使用 Redis 连接器。

集群模式：

FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder()
    .setNodes(new HashSet<InetSocketAddress>(Arrays.asList(new InetSocketAddress(5601)))).build();
DataStream<String> stream = ...;
stream.addSink(new RedisSink<Tuple2<String, String>>(conf, new RedisExampleMapper());
哨兵模式：

FlinkJedisSentinelConfig conf = new FlinkJedisSentinelConfig.Builder()
    .setMasterName("master").setSentinels(...).build();
DataStream<String> stream = ...;
stream.addSink(new RedisSink<Tuple2<String, String>>(conf, new RedisExampleMapper());

---

object JedisUtil {
  def getJedisPool(properties: Properties): JedisPool = {
    val config = new JedisPoolConfig()
config.setBlockWhenExhausted(properties.getProperty("redis.blockWhenExhausted", "true").trim
    ().toBoolean)
	config.setEvictionPolicyClassName(properties.getProperty("redis.evictionPolicyClassName",
      "org.apache.commons.pool2.impl.DefaultEvictionPolicy").trim())


    config.setJmxEnabled(properties.getProperty("redis.jmxEnabled", "true").trim().toBoolean)

    config.setJmxNamePrefix(properties.getProperty("redis.jmxNamePrefix", "pool").trim())

    config.setLifo(properties.getProperty("redis.lifo", "true").trim().toBoolean)

    config.setMaxIdle(properties.getProperty("redis.maxIdle", "8").trim().toInt)

    config.setMaxTotal(properties.getProperty("redis.maxTotal", "8").trim().toInt)
   
    config.setMaxWaitMillis(properties.getProperty("redis.maxWaitMillis", "-1").trim().toInt)
	config.setMinEvictableIdleTimeMillis(properties.getProperty("redis.minEvictableIdleTimeMillis", "1800000").trim().toInt)

    config.setMinIdle(properties.getProperty("redis.minIdle", "0").trim().toInt)
    config.setNumTestsPerEvictionRun(properties.getProperty("redis.numTestsPerEvictionRun", "3")
      .trim().toInt)
    config.setSoftMinEvictableIdleTimeMillis(properties.getProperty("redis.softMinEvictableIdleTimeMillis", "1800000").trim().toInt)

    config.setTestOnBorrow(properties.getProperty("redis.testOnBorrow", "false").trim().toBoolean)

    config.setTestWhileIdle(properties.getProperty("redis.testWhileIdle", "false").trim().toBoolean)

    config.setTimeBetweenEvictionRunsMillis(properties.getProperty("redis.timeBetweenEvictionRunsMillis", "-1").trim().toInt)
    val host = properties.getProperty("redis.host").trim()
    val port = properties.getProperty("redis.port").trim().toInt
    val timeOut = properties.getProperty("redis.timeout").trim().toInt
    val password = properties.getProperty("redis.password").trim()
    new JedisPool(config, host, port, timeOut, password)
  }

}

 jedisPool = JedisUtil.getJedisPool(redisProperties)
 jedis = jedisPool.getResource
         pipeline = jedis.pipelined()
        for (i <- messageArray) {
          //          pipeline.hset("", i(0), i(1))
          //          pipeline.expire("",Int)
          pipeline.set(i(0), i(1), "NX", "EX", redisInPutTTL)
        }
        // pipeline.syncAndReturnAll()
        pipeline.sync()
        messageArray.clear()
        //        jedis.disconnect()