kafka读取本地/hdfs数据向topic发送数据（spark）

最新推荐文章于 2024-08-21 00:07:41 发布

billows9297

最新推荐文章于 2024-08-21 00:07:41 发布

阅读量952

点赞数

文章标签： kafka spark hdfs

本文链接：https://blog.csdn.net/wjt199866/article/details/107055584

版权

代码：

/**
 *
 * @description:
 * @author: wanjintao
 * @time: 2020/6/27 14:57
 *
 */
import java.util.Properties
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.spark.{SparkConf, SparkContext}
object kafka_producer extends Serializable {   //Scala的object和class需要实现接口 Serializable 来序列化

  def main(args: Array[String]) {
    if (args.length < 6) {
      System.err.println("Usage: KafkaProducer <brokers> <topic> " +
        "<inputfile> <interval> <number> <producers>")
      System.exit(1)
    }

    val Array(brokers, topic, inputfile, interval, number, producers) = args

    val conf = new SparkConf().setAppName("kafka_producer").setMaster("local[*]")
    val sc = new SparkContext(conf)
    sc.setLogLevel("ERROR")

    val textfile &