kafka Producer 和 Consumerjava代码实现

最新推荐文章于 2025-03-07 11:08:30 发布

jack-life

最新推荐文章于 2025-03-07 11:08:30 发布

阅读量8.1k

点赞数

分类专栏： kafka&logstash 文章标签： kafka producer consumer

本文链接：https://blog.csdn.net/wslyk606/article/details/78672107

版权

kafka&logstash 专栏收录该内容

4 篇文章

订阅专栏

kafka不同版本是有差别的，我使用的是0.10.0.0版本，从0.8的版本之后，出了一组新的API

1 首先启动kafka

进入kafka目录下的bin目录，使用以下命令启动，&表示后台启动（无zookeeper的话要先启动zookeeper）

.bin/kafka-server-start.sh ./config/server.properties &

启动完成后可以使用 ps -ef | grep kafka 来查看是否启动

2 创建java工程，尽量使用maven

(1) 下面是kafka producer和consumer 低版本Api的java代码

/**
 * 对应低版本的Api
 */
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import java.util.Properties;

public class KafkaProductor {

    public static void main(String[] args) throws InterruptedException {
	
	    //配置kafka参数
        Properties properties = new Properties();
		
		//0.8版本的需要zk.connect
        properties.put("zk.connect", "127.0.0.1:2181");

        properties.put("metadata.broker.list", "localhost:9092");


        properties.put("serializer.class", "kafka.serializer.StringEncoder");


        ProducerConfig producerConfig = new ProducerConfig(properties);

        Producer<String, String> producer = new Producer<>(producerConfig);
		/**
		 * KeyedMessage中"test-topic"为topic的名字，"test-message"为消息内容，也可以加上key，即
		 * new KeyedMessage<>("test-topic","key1" ,"test-message");
		 */
        KeyedMessage<String, String> keyedMessage = new KeyedMessage<>("test-topic", "test-message");

        producer.send(keyedMessage);
        

        Thread.sleep(1000);

        producer.close();

        System.out.println("producer end");
    }
}

(2) 下面是kafka producer和consumer 高版本Api的java代码

/**
 * 对应高版本的Api
 */
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;

public class KafkaProductor {

    public static void main(String[] args) throws InterruptedException {
        Properties properties = new Properties();
		
        properties.put("bootstrap.servers", "127.0.0.1:9092");
      
        properties.put("metadata.broker.list", "localhost:9092");
		
		/**
		 * "key.serializer" 的类型根据ProducerRecord<Integer,String>中的类型来确定，
		 * Integer对应的为IntegerSerializer,String对应的为StringSerializer
		 * key.serializer和value.serializer根据定义的ProducerRecord类型来对应
		 */
        properties.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");


        KafkaProducer kafkaProducer = new KafkaProducer(properties);
		/**
		 * KeyedMessage中"test-topic"为topic的名字，"test-message"为消息内容
		 * 6为对应的key值
		 * "hello"为对应的value值
		 */
        ProducerRecord<Integer,String> producerRecord = new ProducerRecord<>("test-topic", 6,"hello");

        kafkaProducer.send(producerRecord);


        Thread.sleep(1000);

        kafkaProducer.close();

        System.out.println("product end");
    }
}



import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;

import java.util.Collection;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

public class KafkaConsumer {
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put("bootstrap.servers", "127.0.0.1:9092");
        props.put("group.id", "hello-group");
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", "1000");
        props.put("session.timeout.ms", "30000");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.IntegerDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
		
		
        KafkaConsumer<Integer, String> consumer = new KafkaConsumer<>(props);
		
        consumer.subscribe(Collections.singletonList("test-topic"), new ConsumerRebalanceListener() {
            @Override
            public void onPartitionsRevoked(Collection<TopicPartition> partitions) {

            }

            @Override
            public void onPartitionsAssigned(Collection<TopicPartition> partitions) {

            }
        });
		
		
        while (true) {
            ConsumerRecords<Integer, String> records = consumer.poll(100);
            for (ConsumerRecord<Integer, String> record : records) {


                System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(), record.value());
                System.out.println();
            }

            try {
                TimeUnit.MICROSECONDS.sleep(100);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}

3 启动kafka之后，先运行producer，再运行对应的consumer即可消费到kafka消息。

新版本的运行结果为：

offset = 1, key = 6, value = hello

也可以改动key和value的值，offset表示位置，会自动改变。

附上对应的maven pom.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.test</groupId>
    <artifactId>kafka-demo</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-streams</artifactId>
            <version>0.10.0.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>0.10.0.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
            <version>0.9.2</version>
            <exclusions>
                <exclusion>
                    <groupId>org.apache.httpcomponents</groupId>
                    <artifactId>httpclient</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.httpcomponents</groupId>
                    <artifactId>httpcore</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-api</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>com.navercorp.pinpoint</groupId>
            <artifactId>pinpoint-thrift</artifactId>
            <version>1.5.2-SNAPSHOT</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
        <dependency>
            <groupId>commons-codec</groupId>
            <artifactId>commons-codec</artifactId>
            <version>1.9</version>
        </dependency>
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.7</version>
        </dependency>

        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_2.11</artifactId>
            <version>0.10.0.0</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-collections4</artifactId>
            <version>4.0</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>

                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.6.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>