sparkstreaming存入数据redis
!关键 注意使用foreachRDD时,redis连接需要定义在它后面
参照如下代码
否则报错
** * Created by hadoop on 17-5-9. */ import org.apache.spark.streaming.StreamingContext import org.apache.spark.streaming._ import redis.clients.jedis.Jedis import Array._ object test { def main(args: Array[String]): Unit = { // Create a StreamingContext with a local master //从sparkconf创建sparkcontext并指定10秒钟批处理的大小 val ssc = new StreamingContext("local[2]", "NetworkWordCount", Seconds(5)) //连接到本地端口7777后,使用收到的数据创建Dstream val lines = ssc.socketTextStream("localhost", 7777) /* *数据存储到redis中 */ val data=lines.map{wd=> val dataArr:Array[String]=wd.split(" ") val word=dataArr(0) val wordfreqy=dataArr(1) (word,wordfreqy) } if(data!=null) { data.foreachRDD(rdd => { rdd.foreachPartition(partition => { val jedis = new Jedis("127.0.0.1", 6379) partition.foreach(pairs => { jedis.set(pairs._1.toString , pairs._2.toString) jedis.close() }) }) }) } /* * */ // rdd.foreachPartition(partitionOfRecords=>{ // partitionOfRecords.foreach(pair=>{ // val wd=pair._1.toString // val cp=pair._2.toString // if(wd.toString!=null) { // print(wd.toString) // print(cp.toString) // // } // }) // // // }) // // }) // Split each line into words val words = lines.flatMap(_.split(" ")) // Count each word in each batch val pairs = words.map(word => (word, 1)) val wordCounts = pairs.reduceByKey(_ + _) wordCounts.print ssc.start ssc.awaitTermination } }
java读取Redis
注意value格式
涉及到取出数据时的方法
package RedisodataObject; import redis.clients.jedis.Jedis; import java.util.*; /** * Created by hadoop on 17-5-9. */ public class RedisDao { static dataSetObjext data=null; Set<String> k=null; Set<String> v=null; public void testSet(){ //连接redis数据库 Jedis jedis = new Jedis("127.0.0.1", 6379); data=new dataSetObjext(); //将获取的所有value赋值到定义的sethappwords中 data.setStr(jedis.get("难过"));//获取0所有加入的value System.out.println(">>>>>redis中所有的数据key为:"); /* * */ //k =jedis.keys("*"); k=jedis.keys("*"); data.setHappywords(k); System.out.println(data.getHappywords()); //Set循环 System.out.println(">>>>>redis中所有键值对为:"); System.out.println(); for ( String strr :k) { // System.out.println(strr); String key=strr.toString(); String type=jedis.type(key); System.out.println("key的类型为:"+type); String value= jedis.get(key); System.out.println(strr+"->"+value); jedis.close(); } } public static void main(String[] args) { RedisDao rd=new RedisDao(); rd.testSet(); //System.out.println(data.getStr()); } }