GenerateData
package com. domo
import java. io. FileWriter
object GenerateData {
def main ( args: Array[ String] ) : Unit = {
var numCId = 0
var numTableId = 0
var numActualId = 0
val fileName = "E:\\doc\\redis-data1.txt"
val writer: FileWriter = new FileWriter ( fileName, true )
for ( numCid < - 1000000000001 L to 1000000000100 L) {
for ( numTableId < - 1001 to 1050 ) {
for ( numActualId < - 1000000000001 L to 1000000000006 L) {
var valueStr = new StringBuffer ( )
valueStr
. append ( numCid)
. append ( " " )
. append ( numTableId)
. append ( " " )
. append ( numActualId)
. append ( "\n" )
writer. write ( valueStr. toString)
}
}
}
writer. close ( )
}
}
JedisCommand
package com. domo;
import com. domo. util. JedisUtil;
import org. apache. commons. lang3. reflect. MethodUtils;
import redis. clients. jedis. Jedis;
import redis. clients. jedis. Pipeline;
import java. io. IOException;
import java. lang. reflect. Method;
import java. util. ArrayList;
import java. util. List;
public class JedisCommand {
public static List< String> exePipelineCommand ( int partitionCode, String command , List< String> args) throws IOException {
List< String> respList = new ArrayList ( ) ;
Jedis jd = JedisUtil. getJedisByPartitionCode ( partitionCode) ;
Pipeline pl = jd. pipelined ( ) ;
try {
int number = 0 ;
for ( String arg: args) {
String[ ] arr = arg. split ( " " ) ;
pl. zadd ( "CXSZYY:IDX:SECOND2:{" + arr[ 0 ] + "}" , 0 , arr[ 2 ] ) ;
number++ ;
if ( number == 1000 ) {
List< Object> res = pl. syncAndReturnAll ( ) ;
for ( Object o : res) {
respList. add ( o. toString ( ) ) ;
}
number = 0 ;
pl. clear ( ) ;
}
}
List< Object> res = pl. syncAndReturnAll ( ) ;
for ( Object o : res) {
respList. add ( o. toString ( ) ) ;
}
} catch ( Exception e) {
e. printStackTrace ( ) ;
} finally {
pl. close ( ) ;
jd. close ( ) ;
}
return respList;
}
public static void main ( String[ ] args) throws IOException {
Jedis jd = JedisUtil. getJedisByKey ( "exe1" ) ;
List< String> list = new ArrayList < > ( ) ;
list. add ( "exe1 1" ) ;
list. add ( "exe1 2" ) ;
list. add ( "exe5 3" ) ;
List< String> resp = JedisCommand. exePipelineCommand ( JedisUtil. getPartitionCodeByKey ( "exe1" ) , "set" , list) ;
for ( String r: resp) {
System. out. println ( r) ;
}
}
}
JedisUtil
package com. domo. util
import java. util
import java. util. { ArrayList, LinkedHashSet, List}
import org. apache. commons. lang3. reflect. MethodUtils
import redis. clients. jedis. { Client, HostAndPort, Jedis, JedisCluster, JedisPool, JedisPoolConfig, Pipeline, Protocol}
import redis. clients. util. JedisClusterCRC16
import scala. collection. JavaConversions. _
object JedisUtil {
private var jedisCluster: JedisCluster = _
private var jedisPoolConfig: JedisPoolConfig = _
private var nodeMap: util. Map[ String, JedisPool] = _
private var slotHostMap: util. TreeMap[ Long, String] = _
private var clusterSlotsList: util. List[ AnyRef] = _
private var partitionNodeCodeMap: util. HashMap[ String, Int] = _
private var partitionCodeNodeMap: util. HashMap[ Int, String] = _
jedisPoolConfig = new JedisPoolConfig
jedisPoolConfig. setMaxTotal ( 1000 )
jedisPoolConfig. setMaxIdle ( 2 )
jedisPoolConfig. setNumTestsPerEvictionRun ( 1024 )
jedisPoolConfig. setTimeBetweenEvictionRunsMillis ( 30000 )
jedisPoolConfig. setMinEvictableIdleTimeMillis ( 60000 )
jedisPoolConfig. setSoftMinEvictableIdleTimeMillis ( 10000 )
jedisPoolConfig. setMaxWaitMillis ( 30000 )
jedisPoolConfig. setTestOnBorrow ( true )
jedisPoolConfig. setTestWhileIdle ( true )
jedisPoolConfig. setBlockWhenExhausted ( true )
val hostNodes = new util. HashSet [ HostAndPort]
val hostString: String = "192.168.232.132:30001,192.168.232.132:30002,192.168.232.132:30003,192.168.232.132:30004,192.168.232.132:30005,192.168.232.132:30006"
val hostArray: Array[ String] = hostString. split ( "," )
for ( i < - 0 until hostArray. length) {
val hostItems = hostArray ( i)
val hostIpAndPort = hostItems. split ( ":" )
hostNodes. add ( new HostAndPort ( hostIpAndPort ( 0 ) , hostIpAndPort ( 1 ) . toInt) )
}
jedisCluster = new JedisCluster ( hostNodes, 5000 , 5000 , 10 , null, jedisPoolConfig)
nodeMap = jedisCluster. getClusterNodes
val anyHost: String = nodeMap. keySet. iterator. next
clusterSlotsList = getClusterSlots ( anyHost)
slotHostMap = getSlotHostMap
genPartitionMap ( )
private def getClusterSlots ( anyHostAndPortStr: String) = {
var list: util. List[ AnyRef] = null
val parts = anyHostAndPortStr. split ( ":" )
val anyHostAndPort = new HostAndPort ( parts ( 0 ) , parts ( 1 ) . toInt)
try {
val jedis = new Jedis ( anyHostAndPort. getHost, anyHostAndPort. getPort)
val auth = null
if ( auth != null) jedis. auth ( auth)
list = jedis. clusterSlots
jedis. close ( )
} catch {
case e: Exception = >
e. printStackTrace ( )
}
list
}
private def getSlotHostMap = {
val tree = new util. TreeMap [ Long, String]
try {
import scala. collection. JavaConversions. _
for ( obj < - clusterSlotsList) {
val list1 = obj. asInstanceOf[ util. List[ AnyRef] ]
val master = list1. get ( 2 ) . asInstanceOf[ util. List[ AnyRef] ]
val hostAndPort = new String ( master. get ( 0 ) . asInstanceOf[ Array[ Byte] ] ) + ":" + master. get ( 1 )
tree. put ( list1. get ( 0 ) . asInstanceOf[ Long] , hostAndPort)
tree. put ( list1. get ( 1 ) . asInstanceOf[ Long] , hostAndPort)
}
}
catch
{
case e: Exception = >
e. printStackTrace ( )
}
tree
}
private def genPartitionMap ( ) : Unit = {
var partitionNum = 0
partitionNodeCodeMap = new util. HashMap [ String, Int]
partitionCodeNodeMap = new util. HashMap [ Int, String]
try {
for ( obj < - clusterSlotsList) {
val list1 = obj. asInstanceOf[ util. List[ AnyRef] ]
val master = list1. get ( 2 ) . asInstanceOf[ util. List[ AnyRef] ]
val hostAndPort = new String ( master. get ( 0 ) . asInstanceOf[ Array[ Byte] ] ) + ":" + master. get ( 1 )
partitionNodeCodeMap. put ( hostAndPort, partitionNum)
partitionCodeNodeMap. put ( partitionNum, hostAndPort)
partitionNum += 1
}
}
catch
{
case e: Exception = >
e. printStackTrace ( )
}
}
def getRedisTemplate = jedisCluster
def getJedisByKey ( key: String) : Jedis = {
val slot = JedisClusterCRC16. getSlot ( key)
val entry = slotHostMap. lowerEntry ( ( slot + 1 ) . toLong)
var pool: JedisPool = null
var jedis: Jedis = null
try {
pool = nodeMap. get ( entry. getValue)
jedis = pool. getResource
} catch {
case e: RuntimeException = >
e. printStackTrace ( )
if ( jedis != null) jedis. close ( )
}
jedis
}
def getPartitionNumber: Integer = partitionNodeCodeMap. size
def getSlotMap = slotHostMap
def getPartitionCodeByKey ( key: String) : Integer = {
val slot = JedisClusterCRC16. getSlot ( key)
val entry = slotHostMap. lowerEntry ( ( slot + 1 ) . toLong)
partitionNodeCodeMap. get ( entry. getValue)
}
def getPartitionNodeByCode ( code: Int) : String = partitionCodeNodeMap. get ( code)
def getJedisByPartitionCode ( code: Int) : Jedis = {
val pool = nodeMap. get ( getPartitionNodeByCode ( code) )
pool. getResource
}
object Commands {
def fromRedisZrangePipleline ( partitionCode: Int, keys: Array[ String] , tableId: Int, perTime: Int = 1000 )
: java. util. List[ String] = {
if ( keys. length == 0 ) return null
val jd: Jedis = JedisUtil. getJedisByPartitionCode ( partitionCode)
val pipeline: Pipeline = jd. pipelined ( )
var number = 0
val retList: List[ String] = new ArrayList [ String] ( )
try {
keys. foreach ( k = > {
pipeline. zrangeByLex ( k, tableId. toString, ( tableId+ 1 ) . toString)
number += 1
if ( number == perTime) {
val ret: List[ AnyRef] = pipeline. syncAndReturnAll
ret. foreach ( x = > {
retList. addAll ( x. asInstanceOf[ LinkedHashSet[ String] ] )
} )
number = 0
}
} )
val ret: List[ AnyRef] = pipeline. syncAndReturnAll
ret. foreach ( x = > {
retList. addAll ( x. asInstanceOf[ LinkedHashSet[ String] ] )
} )
} catch {
case e: RuntimeException = >
e. printStackTrace ( )
} finally {
pipeline. close ( )
jd. close ( )
}
retList
}
}
def main ( args: Array[ String] ) : Unit = {
}
}
WriteToRedis
package com. domo
import java. util
import com. domo. util. { JedisUtil, RedisRePartition}
import org. apache. spark. { SparkConf, SparkContext, TaskContext}
import scala. collection. JavaConversions. _
object WriteToRedis {
def main ( args: Array[ String] ) : Unit = {
val sparkConf: SparkConf = new SparkConf ( )
. setMaster ( "local[*]" )
. setAppName ( "spark-redis-test" )
val sc: SparkContext = new SparkContext ( sparkConf)
val rdd = sc. textFile ( "E:\\doc\\redis-data1.txt" )
rdd. map ( x= > {
val arr = x. split ( " " )
( arr ( 0 ) , ( arr ( 1 ) + "_" + arr ( 0 ) . reverse + "_" + arr ( 2 ) ) )
} ) . partitionBy ( new RedisRePartition )
. map ( x= > {
( x. _1 + " 0 " + x. _2)
} )
. foreachPartition ( items= > {
val code = TaskContext. getPartitionId ( )
println ( code + ":" + JedisUtil. getPartitionNodeByCode ( code) )
val jd = JedisUtil. getJedisByPartitionCode ( code)
val resp = JedisCommand. exePipelineCommand ( code, "zadd" , items. toList)
resp. foreach ( x= > println ( x) )
} )
}
}