streaming读取kafka数据再保存到es7.7.0代码及踩过的坑

代码

import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.elasticsearch.spark.rdd.EsSpark



object StreamingWriteES {

  def main(args: Array[String]): Unit = {
    // offset保存路径
    // val checkpointPath = "D:\\hadoop\\checkpoint\\kafka-direct"

    val session = SparkSession.builder()
      //.master("local[*]")
      .appName("kafkaStreamingWriteES")
      .config("xpack.security.user","elastic:elastic123")//x-pack认证安全机制的账号密码
      .config("es.net.http.auth.user", "elastic")//es的账号
      .config("es.net.http.auth.pass", "elastic123")//es的密码
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .getOrCreate()
    session.sparkContext.setLogLevel("WARN")

    val ssc = new StreamingContext(session.sparkContext, Seconds(1))
    //  ssc.checkpoint(checkpointPath)

    val bootstrapServers = "xxxx:9092,xxxx:9092,xxxx:9092"
    val groupId = "test-consumer-group3"
    val topicName = "ecar-photo-gps" //
    val maxPoll = 1000

    val kafkaParams = Map(
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServers,
      ConsumerConfig.GROUP_ID_CONFIG -> groupId,
      ConsumerConfig.MAX_POLL_RECORDS_CONFIG -> maxPoll.toString,
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer]
    )

    val kafkaTopicDS = KafkaUtils.createDirectStream(ssc, LocationStrategies.PreferBrokers,
      ConsumerStrategies.Subscribe[String, String](Set(top