八 kafka生产者

kafka官网 http://kafka.apache.org
https://cwiki.apache.org/confluence/display/KAFKA/Clients
查找index 里面包含0.7， 0.8， 0.9， 0.10版本的API

选择相应的版本，用Eclipse打开进行编译项目
例子结合API

kafka启动 cd bin   && ./kafka-server-start.sh ../config/server.properties &
消费者
./kafka-console-consumer.sh --zookeeper hmaster:2181 --topic test --from-beginning
当多个消费者
./kafka-console-consumer.sh --zookeeper hmaster:2181 --topic test

一 HDP启动kafka的端口

[root@kolla spark2]# pwd
/var/log/spark2
[root@kolla spark2]# cat spark-spark-org.apache.spark.deploy.history.HistoryServer-1-kolla.out | grep port
19/01/03 20:07:16 INFO Utils: Successfully started service on port 18081.

二 kafka的命令行目录
[root@kolla spark2]# 
/usr/hdp/2.6.1.0-129/kafka/bin
[root@kolla bin]# 
2.1 创建topic ,进入kafka目录
[root@kolla bin]# ./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
Created topic "test".
2.2验证topic
[root@kolla bin]# ./kafka-topics.sh --list --zookeeper localhost:2181
test
2.3发送消息
[root@kolla bin]# ./kafka-console-producer.sh --broker-list localhost:6667 --topic test

2.4接收消息
[root@kolla bin]# ./kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
Using the ConsoleConsumer with old consumer is deprecated and will be removed in a future major release. Consider using the new consumer by passing [bootstrap-server] instead of [zookeeper].
{metadata.broker.list=kolla:6667, request.timeout.ms=30000, client.id=console-consumer-47216, security.protocol=PLAINTEXT}

实际例子,一个消费者
[root@kolla bin]# ./kafka-console-consumer.sh --zookeeper kolla:2181 --topic test
[root@kolla bin]# ./kafka-console-consumer.sh --zookeeper kolla:2181 --topic test --from-beginning

三eclipse创建工程代码
1先创建一个项目 Eclipse
右键->新建->Maven->Maven Project-> New Maven project (use default Workspace location: /Users/zhangjinyu/Downloads/eclipse-workspace) ->Next
select an Archetype(Group Id : org.apache.maven.archetypes      Artifact Id: maven-archetype-quickstart)  -> Next

Group ID: (com.hzins) Artifact ID:(pro-kf) Version(0.0.1-SNAPSHOT) Package: com.hzins.kafka  ->完成

2创建类，在pro-kf->src/main/java-> com.hzins.kafka->右键->New->Class

其中TestProducer.java 的文件内容为

package com.hzins.kafka;

import java.util.Date;
import java.util.Properties;
import java.util.Random;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

public class TestProducer {

    public static void main(String[] args) {
          long events =10;
            Random rnd = new Random();
     
            Properties props = new Properties();
            props.put("metadata.broker.list", "192.168.229.131:18081");
            props.put("serializer.class", "kafka.serializer.StringEncoder");
            props.put("partitioner.class", "com.hzins.kafka.SimplePartitioner");
            props.put("request.required.acks", "1");
     
            ProducerConfig config = new ProducerConfig(props);
     
            Producer<String, String> producer = new Producer<String, String>(config);

            for (long nEvents = 0; nEvents < events; nEvents++) { 
                   long runtime = new Date().getTime();  
                   String ip = "192.168.2." + rnd.nextInt(255); 
                   String msg = runtime +" www.example.com " + ip; 
                   KeyedMessage<String, String> data = new KeyedMessage<String, String>("test", ip, msg);
               
                   producer.send(data);
                   
            }
            producer.close();
    }
}

SimplePartitioner.java文件内容
package com.hzins.kafka;

import kafka.producer.Partitioner;
import kafka.utils.VerifiableProperties;

public class SimplePartitioner implements Partitioner {
    public SimplePartitioner (VerifiableProperties props) {
 
    }
 
    public int partition(Object key, int a_numPartitions) {
        int partition = 0;
        String stringKey = (String) key;
        int offset = stringKey.lastIndexOf('.');
        if (offset > 0) {
           partition = Integer.parseInt( stringKey.substring(offset+1)) % a_numPartitions;
        }
       return partition;
  }
}

pom.xml文件
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.hzins</groupId>
  <artifactId>pro-kf</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <packaging>jar</packaging>

  <name>pro-kf</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>
    <dependency>
  <groupId>org.apache.kafka</groupId>
  <artifactId>kafka_2.9.2</artifactId>
  <version>0.8.1.1</version>
  <scope>compile</scope>
  <exclusions>
    <exclusion>
      <artifactId>jmxri</artifactId>
      <groupId>com.sun.jmx</groupId>
    </exclusion>
    <exclusion>
      <artifactId>jms</artifactId>
      <groupId>javax.jms</groupId>
    </exclusion>
    <exclusion>
      <artifactId>jmxtools</artifactId>
      <groupId>com.sun.jdmk</groupId>
    </exclusion>
  </exclusions>
</dependency>    
  </dependencies>
</project>