Kafka生产者API

Kafka生产者API,用于将数据发送到 Kafka 集群

代码逻辑：
1. 创建一个KafkaProducer对象
2. 调用KafkaProducer对象的send()方法，将数据发送到Kafka集群
3. 调用KafkaProducer对象的close()方法，关闭KafkaProducer对象

一。从把hdfs文件读取数据
object Test04KafkaProducer {
  def main(args: Array[String]): Unit = {
    val prop = InitKafkaUtil.initPros()
    val producer = new KafkaProducer[Integer, String](prop)
    val path = "hdfs://hacluster/test/ypg/test.txt"
    val fs: FileSystem = HDFSUtil.getHdfs(path)
    val in = fs.open(new Path(path))
    val bufferedReader = new BufferedReader(new InputStreamReader(in))
    var lineTxt: String = bufferedReader.readLine()
    var i = 1
    while (lineTxt != null) {
      println(lineTxt)
      producer.send(new ProducerRecord[Integer, String]("ypg_test", i, lineTxt))
      i += 1
      lineTxt = bufferedReader.readLine()
    }
    bufferedReader.close()
    in.close()
    fs.close()
    producer.close()
  }
}

二.从数据库读取数据
import java.sql.{Connection, DriverManager, ResultSet}

object Test04KafkaProducer {
  def main(args: Array[String]): Unit = {
    val prop = InitKafkaUtil.initPros()
    val producer = new KafkaProducer[Integer, String](prop)

    val url = "jdbc:mysql://localhost:3306/mydb"
    val username = "user"
    val password = "password"
    val connection: Connection = DriverManager.getConnection(url, username, password)
    val statement = connection.createStatement()
    val resultSet: ResultSet = statement.executeQuery("SELECT * FROM mytable")

    var i = 1
    while (resultSet.next()) {
      val data = resultSet.getString("column_name")
      println(data)
      producer.send(new ProducerRecord[Integer, String]("ypg_test", i, data))
      i += 1
    }

    resultSet.close()
    statement.close()
    connection.close()
    producer.close()
  }
}

三。从 API 读取数据
import scala.io.Source
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}

object Test04KafkaProducer {
  def main(args: Array[String]): Unit = {
    val prop = InitKafkaUtil.initPros()
    val producer = new KafkaProducer[Integer, String](prop)

    val url = "https://api.example.com/data"
    val source = Source.fromURL(url)
    val lines = source.getLines()

    var i = 1
    for (line <- lines) {
      println(line)
      producer.send(new ProducerRecord[Integer, String]("ypg_test", i, line))
      i += 1
    }

    source.close()
    producer.close()
  }
}

四。 自定义数据源
object Test04KafkaProducer {
  def main(args: Array[String]): Unit = {
    val prop = new Properties()
    prop.put("bootstrap.servers", "localhost:9092")
    prop.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer")
    prop.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")

    val producer = new KafkaProducer[Long, Long](prop)
    val running = true
    val random = new Random()
    var i = 1

    while (running) {
      val randomNum = random.nextInt(100) + 1
      println(randomNum)
      producer.send(new ProducerRecord[Long,Long]("ypg_test", i, randomNum))
      Thread.sleep(1000)
      i += 1
    }

  }

}