Socket+SparkStreaing+Redis操作代码

sparkstreaming存入数据redis

！关键 注意使用foreachRDD时，redis连接需要定义在它后面

参照如下代码

否则报错

**
  * Created by hadoop on 17-5-9.
  */
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming._
import redis.clients.jedis.Jedis
import Array._

object test {
  def main(args: Array[String]): Unit = {
    // Create a StreamingContext with a local master
    //从sparkconf创建sparkcontext并指定10秒钟批处理的大小
    val ssc = new StreamingContext("local[2]", "NetworkWordCount", Seconds(5))
    //连接到本地端口7777后，使用收到的数据创建Dstream
    val lines = ssc.socketTextStream("localhost", 7777)
    /*
     *数据存储到redis中
     */

     val data=lines.map{wd=>
       val dataArr:Array[String]=wd.split(" ")
       val word=dataArr(0)
       val wordfreqy=dataArr(1)
       (word,wordfreqy)
     }
    if(data!=null) {
      data.foreachRDD(rdd => {
        rdd.foreachPartition(partition => {
          val jedis = new Jedis("127.0.0.1", 6379)
          partition.foreach(pairs => {

            jedis.set(pairs._1.toString , pairs._2.toString)
            jedis.close()

          })
        })
      })
    }

    /*
     *
     */

//      rdd.foreachPartition(partitionOfRecords=>{
//        partitionOfRecords.foreach(pair=>{
//          val wd=pair._1.toString
//          val cp=pair._2.toString
//          if(wd.toString!=null) {
//            print(wd.toString)
//            print(cp.toString)
//
//          }
//        })
//
//
//      })
//
//    })

    // Split each line into words
    val words = lines.flatMap(_.split(" "))
    // Count each word in each batch
    val pairs = words.map(word => (word, 1))
    val wordCounts = pairs.reduceByKey(_ + _)
    wordCounts.print
    ssc.start
    ssc.awaitTermination
  }
}

java读取Redis

注意value格式

涉及到取出数据时的方法

package RedisodataObject;
import redis.clients.jedis.Jedis;
import java.util.*;
/**
 * Created by hadoop on 17-5-9.
 */
public class RedisDao {
    static dataSetObjext data=null;
    Set<String> k=null;
    Set<String> v=null;
    public void testSet(){
        //连接redis数据库
        Jedis jedis = new Jedis("127.0.0.1", 6379);

        data=new dataSetObjext();
        //将获取的所有value赋值到定义的sethappwords中
        data.setStr(jedis.get("难过"));//获取0所有加入的value
        System.out.println(">>>>>redis中所有的数据key为：");
        /*
         *
         */
        //k =jedis.keys("*");
        k=jedis.keys("*");
        data.setHappywords(k);
        System.out.println(data.getHappywords());


        //Set循环
        System.out.println(">>>>>redis中所有键值对为：");
        System.out.println();
        for ( String strr :k) {
          // System.out.println(strr);
           String key=strr.toString();
           String type=jedis.type(key);
            System.out.println("key的类型为："+type);
           String value= jedis.get(key);
            System.out.println(strr+"->"+value);
            jedis.close();
        }

    }
    public static void main(String[] args) {
        RedisDao rd=new RedisDao();
        rd.testSet();
        //System.out.println(data.getStr());
            }
}