package common
import java.util
import redis.clients.jedis.{BinaryJedisCluster, HostAndPort, JedisCluster, JedisPoolConfig}
object RedisDBUtil extends Serializable {
private val nodes:util.Set[HostAndPort] = new util.HashSet[HostAndPort]()
private var connectionTimeOut: Int = 6000
private var soTimeOut: Int = 2000
private var maxAttempts: Int = 3
private var maxTotal: Int = 1000
private var maxIdle: Int = 100
private var minIdle: Int = 0
private var poolConfig: JedisPoolConfig = _
private var password:String = ""
var cluster:JedisCluster = _
def redisCluster(): JedisCluster = {
cluster = new JedisCluster(nodes, this.connectionTimeOut, this.soTimeOut, this.maxAttempts, this.password, this.poolConfig)
cluster
}
def settingRedis(nodes: Array[String],
connectionTimeOut: Int,
soTimeOut: Int,
maxAttempts: Int,
password: String,
maxTotal: Int,
maxIdle: Int,
minIdle: Int): Unit = {
nodes.foreach(node => {
val ipPort = node.split(":")
val port = new HostAndPort(ipPort(0), ipPort(1).toInt)
this.nodes.add(port)
this.connectionTimeOut = connectionTimeOut
this.soTimeOut = soTimeOut
this.maxAttempts = maxAttempts
this.password = password
poolConfig = new JedisPoolConfig()
poolConfig.setMaxTotal(maxTotal)
poolConfig.setMaxIdle(maxIdle)
poolConfig.setMinIdle(minIdle)
})
}
}
def main(args: Array[String]): Unit = {
val conf = new ConfigProperties("D://test.properties")
val sparkConf = new SparkConf().setAppName("app.name").setMaster("local")
sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
val scc = new StreamingContext(sparkConf, Seconds(conf.get("streaming.batch.duration").toLong))
val topics = conf.get("input.kafka.topics").split(",")
val username = conf.get("input.kafka.username")
val password = conf.get("input.kafka.password")
val saslJaasConfig = classOf[PlainLoginModule].getName + " required username=\"" + username + "\" password=\"" + password + "\";"
val kafkaServers = conf.get("input.kafka.bootstrap.servers")
val groupId = conf.get("input.kafka.group.id")
val kafkaParam = Map("bootstrap.servers" -> kafkaServers,
"sasl.jaas.config" -> saslJaasConfig,
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> groupId,
"auto.offset.reset" -> "latest",
"security.protocol" -> "SASL_PLAINTEXT",
"sasl.mechanism" -> "PLAIN")
if (conf.get("input.kafka.session.timeout.ms", null) != null) {
kafkaParam.+("session.timeout.ms" -> conf.get("input.kafka.session.timeout.ms"))
}
if (conf.get("input.kafka.max.poll.interval.ms", null) != null) {
kafkaParam.+("max.poll.interval.ms" -> conf.get("input.kafka.max.poll.interval.ms"))
}
if (conf.get("input.kafka.max.poll.records", null) != null) {
kafkaParam.+("max.poll.records" -> conf.get("input.kafka.max.poll.records"))
}
try {
val nodes = conf.get("output.redis.uri").split(",")
val connectionTimeOut = conf.get("output.redis.connectionTimeout").toInt
val soTimeOut = conf.get("output.redis.soTimeout").toInt
val maxAttempts = conf.get("output.redis.maxAttempts").toInt
val RedisPassword = conf.get("output.redis.password").toString
val maxTotal = conf.get("output.redis.maxTotal").toInt
val maxIdle = conf.get("output.redis.maxIdle").toInt
val minIdle = conf.get("output.redis.minIdle").toInt
RedisDBUtil.settingRedis(nodes, connectionTimeOut, soTimeOut, maxAttempts, RedisPassword, maxTotal, maxIdle, minIdle)
val jedis = RedisDBUtil.redisCluster()
val offsetMap = OffsetUtil.getOffsetMapFromRedis(groupId, topics, jedis)
val kafkaStream = if (offsetMap.size > 0) {
KafkaUtils.createDirectStream[String, String](
scc,
LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](topics, kafkaParam, offsetMap))
} else {
KafkaUtils.createDirectStream[String, String](
scc,
LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](topics, kafkaParam))
}
val kafkaProducer: Broadcast[KafkaSink[String, String]] = {
val username = conf.get("output.kafka.username")
val password = conf.get("output.kafka.password")
val brokers = conf.get("output.kafka.bootstrap.servers")
val kafkaProducerConfig = KafkaFactory.getProducerConfigProperties(brokers, username, password)
println("kafka producer init done!")
scc.sparkContext.broadcast(KafkaSink[String, String](kafkaProducerConfig))
}
val mpprTopics = conf.get("output.kafka.topics.mppr").split(",")
val outputTopicMap = selectTopic(mpprTopics)
kafkaStream.foreachRDD(rdd => {
if (rdd.count() > 0) {
rdd.map(_.value()).map(rawLogStr => {
LogProcessUtil.getLogContentFromKafka(rawLogStr)
}).filter(originLogStr => {
println("原始日志: " + originLogStr)
LogProcessUtil.platLogCheck(originLogStr)
}).map(logStr => {
LogProcessUtil.getPlatformLogJson(logStr)
}).foreach(record => {
try {
println("json字符串日志: " + record)
val recordJson = JSON.parseObject(record)
if (recordJson.get("chlid").equals("1000")) {
kafkaProducer.value.send(outputTopicMap("1000"), record)
saveOffsetForRedis(rdd, groupId)
} else if (recordJson.get("chlid").equals("1001")) {
kafkaProducer.value.send(outputTopicMap("1001"), record)
saveOffsetForRedis(rdd, groupId)
}
} catch {
case e: Throwable => println(e)
}
})
}
})
} catch {
case e: Exception => print(e)
}
scc.start()
scc.awaitTermination()
}
def saveOffsetForRedis(rdd: RDD[ConsumerRecord[String, String]], groupId: String) = {
val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
for (o <- offsetRanges) {
println(s"topic=${o.topic},partition=${o.partition},fromOffset=${o.fromOffset},untilOffset=${o.untilOffset}")
}
OffsetUtil.savaOffsetForResis(groupId, offsetRanges)
}
def saveOffsetForMysql(rdd: RDD[ConsumerRecord[String, String]], groupId: String, mysqlJdbcUrl: String, mysqlUsername: String, mysqlPassword: String) = {
val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
for (o <- offsetRanges) {
println(s"topic=${o.topic},partition=${o.partition},fromOffset=${o.fromOffset},untilOffset=${o.untilOffset}")
}
OffsetUtil.saveOffsetforMysql(groupId, offsetRanges, mysqlJdbcUrl, mysqlUsername, mysqlPassword)
}
def selectTopic(topicArray: Array[String]): Map[String, String] = {
var resultMap: Map[String, String] = new HashMap[String, String]()
topicArray.foreach(topic => {
val topicArray = topic.split("_").toArray
resultMap += (topicArray(5) -> topic)
})
return resultMap
}
package common
import java.sql.{DriverManager, ResultSet}
import org.apache.kafka.common.TopicPartition
import org.apache.spark.streaming.kafka010.OffsetRange
import redis.clients.jedis.{BinaryJedisCluster, JedisCluster}
import scala.collection.JavaConversions._
import scala.collection.mutable
object OffsetUtil {
def getOffsetMapFromMysql(groupId: String, topics: Array[String], mysqlJdbcUrl: String, mysqlUsername: String, mysqlPassword: String) = {
val connection = DriverManager.getConnection(mysqlJdbcUrl, mysqlUsername, mysqlPassword)
val pstmt = connection.prepareStatement("select `topic`,`partition`,`offset` from dataServer_offset where `groupId`=? and `topic`=?")
val offsetMap = new mutable.HashMap[TopicPartition, Long]()
var result: ResultSet = null
topics.foreach(topic => {
pstmt.setString(1, groupId)
pstmt.setString(2, topic)
result = pstmt.executeQuery()
while (result.next()) {
offsetMap += new TopicPartition(result.getString("topic"), result.getInt("partition")) -> result.getLong("offset")
}
})
result.close()
pstmt.close()
connection.close()
offsetMap
}
def saveOffsetforMysql(groupid: String, offsetRange: Array[OffsetRange], mysqlJdbcUrl: String, mysqlUsername: String, mysqlPassword: String) = {
val connection = DriverManager.getConnection(mysqlJdbcUrl, mysqlUsername, mysqlPassword)
val pstmt = connection.prepareStatement("replace into dataServer_offset(`topic`, `partition`, `groupId`, `offset`) values(?,?,?,?)")
offsetRange.foreach(o => {
pstmt.setString(1, o.topic)
pstmt.setInt(2, o.partition)
pstmt.setString(3, groupid)
pstmt.setLong(4, o.untilOffset)
pstmt.executeUpdate()
})
pstmt.close()
connection.close()
}
def getOffsetMapFromRedis(groupId: String, topics: Array[String], jedis: JedisCluster) = {
val offsetMap = new mutable.HashMap[TopicPartition, Long]()
topics.foreach(topic => {
val resultMap = jedis.hgetAll(topic)
for((groupAndPartition:String, offsetStr:String) <- resultMap){
val gps = groupAndPartition.split("\\|")
val redisGroup = gps(0)
if(redisGroup == groupId) {
val partition = gps(1).toInt
val offset = offsetStr.toLong
offsetMap += new TopicPartition(topic, partition) -> offset
}
}
})
jedis.close()
offsetMap
}
def savaOffsetForResis(groupid: String, offsetRange: Array[OffsetRange]) = {
val jedis = RedisDBUtil.cluster
for(offsetRange <- offsetRange) {
val topic = offsetRange.topic
val partition = offsetRange.partition
val offset = offsetRange.untilOffset
val topicAndPartition = s"${groupid}|${partition}"
jedis.hset(topic, topicAndPartition, offset + "")
}
jedis.close()
}
}