val driver ="com.mysql.jdbc.Driver"
val url ="jdbc:mysql://aliyun01:3306/kafka"
val userName ="root"
val passWd ="root"
def getOffsetFromMysql(groupId:String,topicName:String): util.Map[TopicPartition, java.lang.Long]={
Class.forName(driver)
val conn: Connection = DriverManager.getConnection(url, userName, passWd)
val sql="select `groupid`,`topic`,`partition`,`untiloffset` from offset_manager where groupid=? and topic=?"
val pre = conn.prepareStatement(sql)
pre.setString(1,s"${groupId}")
pre.setString(2,s"${topicName}")
val resultSet: ResultSet = pre.executeQuery()
val resMap: util.Map[TopicPartition, java.lang.Long]=newutil.HashMap[TopicPartition,java.lang.Long]()while(resultSet.next()){
val topicPartition: TopicPartition =newTopicPartition(resultSet.getString("topic"),resultSet.getInt("partition"))
resMap .put(topicPartition,resultSet.getString("untiloffset").toLong)}
resMap
}
def getOffsetFromMysql(groupId:String,topicName:String): util.Map[TopicPartition, java.lang.Long]={
val conn = SqlDataSource.getConnection()
val sql="select `groupid`,`topic`,`partition`,`untiloffset` from offset_manager where groupid=? and topic=?"
val pre = conn.prepareStatement(sql)
pre.setString(1,s"${groupId}")
pre.setString(2,s"${topicName}")
val resultSet: ResultSet = pre.executeQuery()
val resMap: util.Map[TopicPartition, java.lang.Long]=newutil.HashMap[TopicPartition,java.lang.Long]()while(resultSet.next()){
val topicPartition: TopicPartition =newTopicPartition(resultSet.getString("topic"),resultSet.getInt("partition"))
resMap .put(topicPartition,resultSet.getString("untiloffset").toLong)}
SqlDataSource.closeResultSet(resultSet)
SqlDataSource.closePrepareStatement(pre)
SqlDataSource.closeConnection(conn)
resMap
}}
五维护偏移量至mysql
def updateOffset(kafkaDataDstream:InputDStream[ConsumerRecord[String, String]],groupId:String)={
kafkaDataDstream.foreachRDD(rdd=>{
val conn = SqlDataSource.getConnection()//获取连接
val sql: String ="replace into `offset_manager` (groupid,topic,`partition`,untilOffset) values(?,?,?,?)"
val pre: PreparedStatement = conn.prepareStatement(sql)
conn.setAutoCommit(false)
val offsetRanges: Array[OffsetRange]= rdd.asInstanceOf[HasOffsetRanges].offsetRanges
for(offset <-offsetRanges){
val partition = offset.partition // 分区号
val topic = offset.topic // 主题//val fromOffset = offset.fromOffset // 起始偏移量
val endOffset = offset.untilOffset // 结束偏移量
pre.setString(1,groupId);
pre.setString(2,topic);
pre.setInt(3,partition);
pre.setString(4,endOffset.toString)
pre.addBatch()}
pre.executeBatch()
conn.commit()
SqlDataSource.closePrepareStatement(pre)
SqlDataSource.closeConnection(conn);})}