mysql管理kafka偏移量

import java.lang
import Utils.OffsetManager
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext}

//mysql管理偏移量
object SSCDirectKafka010_Mysql_Offset {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName("SSCDirectKafka010_Mysql_Offset")

    //配置在kafka中每次拉取的数据量,这里配置的2并不是每次在kafka拉取2条数据,而是:2*分区数量*采样时间(12)
    conf.set("spark.streaming.kafka.maxRatePerPartition", "2")
    //是否优雅的停止你的SparkStreaming,如果不加这个参数的话,服务停止的时候可能会造成数据的丢失
    conf.set("spark.streaming.stopGracefullyOnShutdown", "true")

    val ssc = new StreamingContext(conf,Seconds(2))

    //消费者ID
    val groupId = "day11_07"

    //设置消费者参数
    val kafkaParams = Map[String, Object](
      "bootstrap.servers" -> "hadoop01:9092,hadoop02:9092,hadoop03:9092",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> groupId,
      //从头的数据开始消费earliest
      "auto.offset.reset" -> "earliest",
      //是否自动提交偏移量
      "enable.auto.commit" -> (false: lang.Boolean)
    )

    //指定主题
    val topic = "helloTopic"
    val topics: Array[String] = Array(topic)


    //可以传多个主题(这里是一个)
    val offsetManager = OffsetManager(groupId,topic)

    val result = if(offsetManager.size>0){
      KafkaUtils.createDirectStream[String,String](
        ssc,
        LocationStrategies.PreferConsistent,
        ConsumerStrategies.Subscribe[String,String](topics,kafkaParams,offsetManager)
      )
    }else{
      KafkaUtils.createDirectStream[String,String](
        ssc,
        LocationStrategies.PreferConsistent,
        ConsumerStrategies.Subscribe[String,String](topics,kafkaParams)
      )
    }

    result.foreachRDD(rdd=>{
      //获取当前偏移量
      val ranges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
      ranges.foreach(println(_))

      //管理偏移量 sql和redis
      //手动提交偏移量
      //result.asInstanceOf[CanCommitOffsets].commitAsync(ranges)

      //将偏移量存入mysql
      OffsetManager.saveCurrentBatchOffset(groupId,ranges)
    })

    ssc.start()

    ssc.awaitTermination()

  }
}

工具Utils

import java.sql.DriverManager
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.kafka.common.TopicPartition
import org.apache.spark.streaming.kafka010.OffsetRange

object OffsetManager {

  //读取配置文件
  val config: Config = ConfigFactory.load()
  //数据库连接参数配置
  def getConn = {
    DriverManager.getConnection(
      config.getString("db.url"),
      config.getString("db.user"),
      config.getString("db.password")
    )
  }


  /*
  获取偏移量信息
   */
  def apply(groupId: String,topic:String) = {
    val conn = getConn
    val statement = conn.prepareStatement("select * from kafka_offset where groupId=? and topic=?")
    statement.setString(1,groupId)
    statement.setString(2,topic)
    val rs = statement.executeQuery()
    //注意导包
    import scala.collection.mutable._
    val offsetRange = Map[TopicPartition,Long]()
    while (rs.next()){

      //讲获取的数据放到map中
      offsetRange += new TopicPartition(rs.getString("topic"),rs.getInt("partition")) -> rs.getLong("untilOffset")
    }
    rs.close()
    statement.close()
    conn.close()
    offsetRange
  }

  /*
  保存当前偏移量到数据库
   */
  def saveCurrentBatchOffset(groupId:String,offsetRange:Array[OffsetRange]) = {
    val conn = getConn
    val statement = conn.prepareStatement("replace into kafka_offset values(?,?,?,?)")
    for (i <- offsetRange){
      statement.setString(1,i.topic)
      statement.setInt(2,i.partition)
      statement.setLong(3,i.untilOffset)
      statement.setString(4,groupId)
      statement.executeUpdate()

    }

    statement.close()

    conn.close()

  }

}

  • 1
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
要获取Kafka偏移量,可以使用Kafka Consumer API提供的方法。以下是获取Kafka偏移量的示例代码: ```java import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; import java.util.Arrays; import java.util.Properties; public class KafkaOffsetExample { private static final String TOPIC_NAME = "my-topic"; private static final String BOOTSTRAP_SERVERS = "localhost:9092"; private static final String GROUP_ID = "my-group"; public static void main(String[] args) { Properties props = new Properties(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props); // 订阅主题 consumer.subscribe(Arrays.asList(TOPIC_NAME)); // 获取分区 TopicPartition partition = new TopicPartition(TOPIC_NAME, 0); // 获取偏移量 long offset = consumer.position(partition); System.out.println("Offset: " + offset); // 关闭消费者 consumer.close(); } } ``` 在这个例子中,我们使用KafkaConsumer类获取Kafka偏移量。首先,我们创建KafkaConsumer实例并订阅主题。然后,我们获取分区并使用position()方法获取当前偏移量。最后,我们打印偏移量并关闭消费者。

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值