Avro格式

import org.apache.spark.sql.avro.{SchemaConverters, AvroSerializer}

import io.confluent.kafka.serializers.KafkaAvroSerializer

import org.apache.avro.Schema

import org.apache.avro.generic.{GenericRecord, GenericData}

import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord, ProducerConfig}

import java.util.Properties

 

val avroSchema: Schema = SchemaConverters.toAvroType(df.schema)

 

val producerProps = new Properties()

producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")

producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "io.confluent.kafka.serializers.KafkaAvroSerializer")

producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "io.confluent.kafka.serializers.KafkaAvroSerializer")

producerProps.put("schema.registry.url", "http://localhost:8081")

 

val producer