RedisCode

GenerateData

package com.domo

import java.io.FileWriter

object GenerateData {
  def main(args: Array[String]): Unit = {

    var numCId = 0
    var numTableId = 0
    var numActualId = 0

    val fileName = "E:\\doc\\redis-data1.txt"


    val writer: FileWriter = new FileWriter(fileName, true)

    for (numCid <- 1000000000001L to 1000000000100L) {
      for (numTableId <- 1001 to 1050) {

        for (numActualId <- 1000000000001L to 1000000000006L) {
          var valueStr = new StringBuffer()
          valueStr
            .append(numCid)
            .append(" ")
            .append(numTableId)
            .append(" ")
            .append(numActualId)
            .append("\n")

          writer.write(valueStr.toString)

        }
      }
    }

    writer.close()
  }
}

JedisCommand

package com.domo;

import com.domo.util.JedisUtil;
import org.apache.commons.lang3.reflect.MethodUtils;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.Pipeline;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;

public class JedisCommand {

    public static List<String> exePipelineCommand(int partitionCode, String command , List<String> args) throws IOException {



        List<String> respList = new ArrayList();
        Jedis jd = JedisUtil.getJedisByPartitionCode(partitionCode);
        Pipeline pl = jd.pipelined();

        try {
            int number = 0;
            for (String arg: args) {
//                System.out.println(arg);
                String[] arr = arg.split(" ");
//                method.invoke(pl, arr[0],Double.valueOf(arr[1]), arr[2]);
                pl.zadd("CXSZYY:IDX:SECOND2:{" + arr[0] + "}", 0, arr[2]);
                number++;
                if (number == 1000) {
                    List<Object> res = pl.syncAndReturnAll();
                    for (Object o : res) {
                        respList.add(o.toString());
                    }
                    number = 0;
                    pl.clear();
                }
            }
            List<Object> res = pl.syncAndReturnAll();
            for (Object o : res) {
                respList.add(o.toString());
            }

        }catch(Exception e) {
            e.printStackTrace();
        }finally {
            pl.close();
            jd.close();
        }
        return respList;

    }



    public static void main(String[] args) throws IOException {
        Jedis jd = JedisUtil.getJedisByKey("exe1");
        List<String> list  = new ArrayList<>();

        list.add("exe1 1");
        list.add("exe1 2");
        list.add("exe5 3");


        List<String> resp = JedisCommand.exePipelineCommand(JedisUtil.getPartitionCodeByKey("exe1"), "set" , list);

        for(String r: resp){
            System.out.println(r);
        }
    }
}

JedisUtil

package com.domo.util

import java.util
import java.util.{ArrayList, LinkedHashSet, List}

import org.apache.commons.lang3.reflect.MethodUtils
import redis.clients.jedis.{Client, HostAndPort, Jedis, JedisCluster, JedisPool, JedisPoolConfig, Pipeline, Protocol}
import redis.clients.util.JedisClusterCRC16

import scala.collection.JavaConversions._


object JedisUtil {



  private var jedisCluster: JedisCluster = _
  private var jedisPoolConfig: JedisPoolConfig = _
  private var nodeMap: util.Map[String, JedisPool] = _
  private var slotHostMap: util.TreeMap[Long, String] = _
  private var clusterSlotsList: util.List[AnyRef] = _
  private var partitionNodeCodeMap: util.HashMap[String, Int] = _
  private var partitionCodeNodeMap: util.HashMap[Int, String] = _

  jedisPoolConfig = new JedisPoolConfig
//  jedisPoolConfig.setMaxTotal(PropertiesUtil.getValues("redis.pool.maxTotal").toInt)
//  jedisPoolConfig.setMaxIdle(PropertiesUtil.getValues("redis.pool.maxIdle").toInt)
//  jedisPoolConfig.setNumTestsPerEvictionRun(PropertiesUtil.getValues("redis.pool.numTestsPerEvictionRun").toInt)
//  jedisPoolConfig.setTimeBetweenEvictionRunsMillis(PropertiesUtil.getValues("redis.pool.timeBetweenEvictionRunsMillis").toLong)
//  jedisPoolConfig.setMinEvictableIdleTimeMillis(PropertiesUtil.getValues("redis.pool.minEvictableIdleTimeMillis").toLong)
//  jedisPoolConfig.setSoftMinEvictableIdleTimeMillis(PropertiesUtil.getValues("redis.pool.softMinEvictableIdleTimeMillis").toLong)
//  jedisPoolConfig.setMaxWaitMillis(PropertiesUtil.getValues("redis.pool.maxWaitMillis").toLong)
//  jedisPoolConfig.setTestOnBorrow(PropertiesUtil.getValues("redis.pool.testOnBorrow").toBoolean)
//  jedisPoolConfig.setTestWhileIdle(PropertiesUtil.getValues("redis.pool.testWhileIdle").toBoolean)
//  jedisPoolConfig.setBlockWhenExhausted(PropertiesUtil.getValues("redis.pool.blockWhenExhausted").toBoolean)

  jedisPoolConfig.setMaxTotal(1000)
  jedisPoolConfig.setMaxIdle(2)
  jedisPoolConfig.setNumTestsPerEvictionRun(1024)
  jedisPoolConfig.setTimeBetweenEvictionRunsMillis(30000)
  jedisPoolConfig.setMinEvictableIdleTimeMillis(60000)
  jedisPoolConfig.setSoftMinEvictableIdleTimeMillis(10000)
  jedisPoolConfig.setMaxWaitMillis(30000)
  jedisPoolConfig.setTestOnBorrow(true)
  jedisPoolConfig.setTestWhileIdle(true)
  jedisPoolConfig.setBlockWhenExhausted(true)

  val hostNodes = new util.HashSet[HostAndPort]
//  val hostString: String = PropertiesUtil.getValues("redis.nodes")
  val hostString: String = "192.168.232.132:30001,192.168.232.132:30002,192.168.232.132:30003,192.168.232.132:30004,192.168.232.132:30005,192.168.232.132:30006"
  val hostArray: Array[String] = hostString.split(",")

  for (i <- 0 until hostArray.length) {
    val hostItems = hostArray(i)
    val hostIpAndPort = hostItems.split(":")
    hostNodes.add(new HostAndPort(hostIpAndPort(0), hostIpAndPort(1).toInt))
  }

  jedisCluster = new JedisCluster(hostNodes, 5000, 5000, 10, null, jedisPoolConfig)
  nodeMap = jedisCluster.getClusterNodes
  val anyHost: String = nodeMap.keySet.iterator.next
  clusterSlotsList = getClusterSlots(anyHost)
  slotHostMap = getSlotHostMap
  genPartitionMap()


  private def getClusterSlots(anyHostAndPortStr: String) = {
    var list: util.List[AnyRef] = null
    val parts = anyHostAndPortStr.split(":")
    val anyHostAndPort = new HostAndPort(parts(0), parts(1).toInt)
    try {
      val jedis = new Jedis(anyHostAndPort.getHost, anyHostAndPort.getPort)
//      val auth = PropertiesUtil.getValues("redis.password")
      val auth = null
      if (auth != null) jedis.auth(auth)
      list = jedis.clusterSlots
      jedis.close()
    } catch {
      case e: Exception =>
        e.printStackTrace()
    }
    list
  }

  private def getSlotHostMap = {
    val tree = new util.TreeMap[Long, String]
    try {
      import scala.collection.JavaConversions._
      for (obj <- clusterSlotsList) {
        val list1 = obj.asInstanceOf[util.List[AnyRef]]
        val master = list1.get(2).asInstanceOf[util.List[AnyRef]]
        val hostAndPort = new String(master.get(0).asInstanceOf[Array[Byte]]) + ":" + master.get(1)
        tree.put(list1.get(0).asInstanceOf[Long], hostAndPort)
        tree.put(list1.get(1).asInstanceOf[Long], hostAndPort)
      }
    }
    catch
    {
      case e: Exception =>
        e.printStackTrace()
    }
    tree
  }

  private def genPartitionMap(): Unit = {
    var partitionNum = 0
    partitionNodeCodeMap = new util.HashMap[String, Int]
    partitionCodeNodeMap = new util.HashMap[Int, String]
    try {

      for (obj <- clusterSlotsList) {
        val list1 = obj.asInstanceOf[util.List[AnyRef]]
        val master = list1.get(2).asInstanceOf[util.List[AnyRef]]
        val hostAndPort = new String(master.get(0).asInstanceOf[Array[Byte]]) + ":" + master.get(1)
        partitionNodeCodeMap.put(hostAndPort, partitionNum)
        partitionCodeNodeMap.put(partitionNum, hostAndPort)
        partitionNum += 1
      }
    }
    catch
    {
      case e: Exception =>
        e.printStackTrace()
    }
  }

  def getRedisTemplate = jedisCluster

  def getJedisByKey(key: String): Jedis = {
    val slot  = JedisClusterCRC16.getSlot(key)
    // 获取到对应的Jedis对象
    val entry  = slotHostMap.lowerEntry((slot + 1).toLong)
    var pool: JedisPool  = null
    var jedis: Jedis  = null
    try {
      pool = nodeMap.get(entry.getValue)
      jedis = pool.getResource
    } catch {
      case e: RuntimeException =>
        e.printStackTrace()
        if (jedis != null) jedis.close()// 获取连接失败时，应该返回给pool,否则每次发生异常将导致一个jedis对象没有被回收。
    }
    jedis
  }

  def getPartitionNumber: Integer = partitionNodeCodeMap.size

  def getSlotMap = slotHostMap

  def getPartitionCodeByKey(key: String): Integer = {
    val slot = JedisClusterCRC16.getSlot(key)
    val entry = slotHostMap.lowerEntry((slot + 1).toLong)
    partitionNodeCodeMap.get(entry.getValue)
  }

  def getPartitionNodeByCode(code: Int): String = partitionCodeNodeMap.get(code)


  def getJedisByPartitionCode(code: Int): Jedis = {
    val pool = nodeMap.get(getPartitionNodeByCode(code))
    pool.getResource

  }



  object Commands {

    def fromRedisZrangePipleline(partitionCode: Int, keys: Array[String],tableId: Int, perTime: Int = 1000)
    : java.util.List[String] = {

      if (keys.length == 0) return null
      val jd: Jedis = JedisUtil.getJedisByPartitionCode(partitionCode)
      val pipeline: Pipeline = jd.pipelined()
      var number = 0

      val retList: List[String] = new ArrayList[String]()

      try {
        keys.foreach(k => {
//          pipeline.zrange(k, Long.MinValue, Long.MaxValue)
          pipeline.zrangeByLex(k,tableId.toString, (tableId+1).toString)
          number += 1
          if (number == perTime) {
            val ret: List[AnyRef] = pipeline.syncAndReturnAll
            ret.foreach(x => {
              retList.addAll(x.asInstanceOf[LinkedHashSet[String]])
            })
            number = 0
          }
        })

        val ret: List[AnyRef] = pipeline.syncAndReturnAll
        ret.foreach(x => {
          retList.addAll(x.asInstanceOf[LinkedHashSet[String]])
        })

      } catch {
      case e: RuntimeException =>
        e.printStackTrace()
    } finally {
      pipeline.close()
      jd.close()
    }
    retList
    }


  }

  def main(args: Array[String]): Unit = {
//    val jd = JedisUtil.getJedisByKey("exe1")
//    JedisUtil.execRedisCommand(jd, "set" , "exe1 1")

  }


}

WriteToRedis

package com.domo

import java.util

import com.domo.util.{JedisUtil, RedisRePartition}
import org.apache.spark.{SparkConf, SparkContext, TaskContext}

import scala.collection.JavaConversions._

object WriteToRedis {

  def main(args: Array[String]): Unit = {
   val sparkConf: SparkConf = new SparkConf()
      //      .setMaster("spark://192.168.232.129:7077")
      .setMaster("local[*]")
      .setAppName("spark-redis-test")

    val sc: SparkContext = new SparkContext(sparkConf)

    val rdd = sc.textFile("E:\\doc\\redis-data1.txt")
    rdd.map(x=>{
      val arr = x.split(" ")
      (arr(0),(arr(1) + "_" + arr(0).reverse + "_" + arr(2)))
    }).partitionBy(new RedisRePartition)
      .map(x=>{
        (x._1  + " 0 " + x._2)
      })
      .foreachPartition(items=>{

        val code = TaskContext.getPartitionId()
        println(code +":"+JedisUtil.getPartitionNodeByCode(code))
        val jd = JedisUtil.getJedisByPartitionCode(code)

        val resp = JedisCommand.exePipelineCommand(code, "zadd", items.toList)

       resp.foreach(x=>println(x))

      })

  }

}