Spark实时项目第三天-精准一次消费|手动提交偏移

精确一次消费

问题产生

解决方案

手动提交偏移量

用Redis保存偏移量原因

编写OffsetManagerUtil

在scala\com\atguigu\gmall\realtime\utils\OffsetManagerUtil

import java.util

import org.apache.kafka.common.TopicPartition
import org.apache.spark.streaming.kafka010.OffsetRange
import redis.clients.jedis.Jedis
import  scala.collection.JavaConversions._
object OffsetManagerUtil {
   


  /**
   * 从Redis中读取偏移量
   * @param groupId
   * @param topic
   * @return
   */
  def getOffset(groupId:String,topic:String):Map[TopicPartition,Long]={
   
    var offsetMap=Map[TopicPartition,Long]()

    val jedisClient: Jedis = RedisUtil.getJedisClient

    val redisOffsetMap: util.Map[String, String] = jedisClient.hgetAll("offset:"+groupId+":"+topic)
    jedisClient.close()
    if(redisOffsetMap!=null&&redisOffsetMap.isEmpty){
   
      null
    }else {
   

      val redisOffsetList: List[(String, String)] = redisOffsetMap.toList

      val kafkaOffsetList: List[(TopicPartition, Long)] = redisOffsetList.map {
    case ( partition, offset) =>
        (new TopicPartition(topic, partition.toInt), offset.toLong)
      }
      kafkaOffsetList.toMap
    }
  }

  /**
   * 偏移量写入到Redis中
   * @param groupId
   * @param topic
   * @param offsetArray
   */
  def saveOffset(groupId:String,topic:String ,offsetArray:Array[OffsetRange]):Unit= {
   
    if (offsetArray != null && offsetArray.size >