Kafka生产者和消费者的简单应用——01生产者

maven工程pom文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>SparkStreaming</groupId>
    <artifactId>strraming</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <scala.version>2.11.8</scala.version>
        <spark.version>2.1.2</spark.version>
        <hadoop.version>2.7.1</hadoop.version>
        <encoding>UTF-8</encoding>
    </properties>

    <dependencies>
        <!--sparkjar包-->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <!--kafka的客户端jar包-->
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>0.10.2.0</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <!-- 指定编译scala的插件 -->
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                        <configuration>
                            <args>
                                <arg>-dependencyfile</arg>
                                <arg>${project.build.directory}/.scala_dependencies</arg>
                            </args>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

        </plugins>
    </build>
</project>

生产者代码实现

import java.util.Properties

import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.apache.kafka.common.serialization.StringSerializer

/**
  * Created by X230 on 2018/9/12.
  */
object KafkaProducerApi {
//要是不想看log4j 中那么多的日志信息可以屏蔽 我不在乎这些哈哈
//private var level: Unit = Logger.getLogger("org").setLevel(Level.WARN)
  def main(args: Array[String]): Unit = {

    var props: Properties = new Properties()
    //在集群中指定Broker的地址，这里自己在window中设置了映射（没设置的可以写IP地址）
    props.setProperty("bootstrap.servers","cent01:9092,cent02:9092,cent03:9092")
    //设置Key的序列化类型，这里只是简单的操作字符串 String
    props.setProperty("key.serializer",classOf[StringSerializer].getName)
    //设置value的序列化类型
    props.setProperty("value.serializer",classOf[StringSerializer].getName)
    //创建生产者的实例对象
    var producer: KafkaProducer[String, String] = new KafkaProducer[String,String](props)
    var i=1
    //简单的生产数据，因为数据不可能是一条 运用循环实现简单的模拟生产，觉得没什么感觉可以在循环中sleep几秒
    while(i<=100){
       //为了直观的展现分区在不同broker的情况 我这里是3台机器 就设置了3个分区 当然也可以设置多个 自己爱好，
      val partition =i%3
      //发送消息 使用ProducerRecord来封装，四个参数：topic，partition，key，value
      var record: ProducerRecord[String, String] = new ProducerRecord[String,String]("helloTopic",partition,"",i+"hello")

      producer.send(record)
      println(record)
      i+=1
    }
    //为了查看是否循环结束
    println("success")
  }
}