Spark-Redis

pom依赖：

        <!-- redis依赖 -->
        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
            <version>2.9.0</version>
        </dependency>

package spark.streaming
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import redis.clients.jedis.Jedis

object streamingRedis {

  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("streamingRedis").setMaster("local[2]")
    val ssc = new StreamingContext(conf,Seconds(5))

    //创建文本输入流，并进行词频统计
    val lines: ReceiverInputDStream[String] = ssc.socketTextStream("hdp-1",6666)
    val pairs: DStream[(String, Int)] = lines.flatMap(_.split(",")).map(x => (x,1)).reduceByKey(_+_)
    //保存数据到Redis
    pairs.foreachRDD { rdd =>
      rdd.foreachPartition { partitionOfRecords =>
        var jedis: Jedis = null
        try {
          jedis = JedisPoolUtil.getConnection
          jedis.auth("1234")
          partitionOfRecords.foreach(record => jedis.hincrBy("wordCount", record._1, record._2))
        } catch {
          case ex: Exception =>
            ex.printStackTrace()
        } finally {
          if (jedis != null) jedis.close()
        }
      }
    }
    ssc.start()
    ssc.awaitTermination()

    }



}

package spark.streaming;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;

public class JedisPoolUtil {
    //声明为volatile防止指令重排序
    private static volatile JedisPool jedisPool = null;
    private static volatile String HOST = "hdp-1";
    private static volatile int PORT = 6379;

    //双重检查锁实现懒汉模式单例
    public static Jedis getConnection() {
        if (jedisPool == null) {
            synchronized (JedisPoolUtil.class) {
                if (jedisPool == null) {
                    JedisPoolConfig config = new JedisPoolConfig();
                    config.setMaxTotal(30);
                    config.setMaxIdle(10);
                    jedisPool = new JedisPool(config, HOST, PORT);
                }
            }
        }
        return jedisPool.getResource();
    }

}

运行完后结果会保存到Redis中