Kafka使用入门

最新推荐文章于 2024-10-03 08:04:53 发布

MarsZhangLing

最新推荐文章于 2024-10-03 08:04:53 发布

阅读量1.3k

点赞数

分类专栏： java 文章标签： java kafka

本文链接：https://blog.csdn.net/makefriend7/article/details/71405343

版权

java 专栏收录该内容

20 篇文章 0 订阅

订阅专栏

首先依然放官网地址http://kafka.apache.org/ 然后是本人的下载地址和版本 https://www.apache.org/dyn/closer.cgi?path=/kafka/0.10.2.1/kafka_2.12-0.10.2.1.tgz

RocketMq 和 Kafka到底使用哪个，确实是蛮令人纠结的，我个人查询的结论就是，如果你想方便的使用，那就用RocketMq,如果你想追求高性能，那就使用Kafka（好吧，感觉再高也是高不过ZMQ的，不过ZMQ几乎啥事情都没帮我们干），就目前而言。 kafka的文档看起来更舒服一些，这也使我决定先试用下kafka。

首先是启动

.\bin\windows\zookeeper-server-start .\config\zookeeper.properties

.\bin\windows\kafka-server-start .\config\server.properties

装了JAVA 32位，却用了WIN 64位的奇葩，修改kafka-server-start.bat

set KAFKA_HEAP_OPTS=-Xmx400M -Xms400M

搞个类

package util;

import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;

import java.util.HashMap;
import java.util.Map;

public class KafkaUtil {
    private static Map<String, Object> senderProps(String strIpPort) {
        Map<String, Object> props = new HashMap<>();
        //  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.1.1:9092");
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, strIpPort);
        props.put(ProducerConfig.RETRIES_CONFIG, 0);
        props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
        props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        return props;
    }
    public static KafkaTemplate<Integer, String>  initSender(String strIpPort)
    {
        Map<String, Object> senderProps = senderProps(strIpPort);
        ProducerFactory<Integer, String> pf =
                new DefaultKafkaProducerFactory<Integer, String>(senderProps);
        KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
        return template;
    }


}

测试类通过

package util;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.serialization.IntegerDeserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.junit.Test;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.kafka.listener.MessageListener;
import org.springframework.kafka.listener.config.ContainerProperties;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;

class MyListener implements MessageListener<Integer, String>
{
    @Override
    public void onMessage(ConsumerRecord<Integer, String> data) {
        System.out.println("received: " + data);
        System.out.println("received: String   " + data.toString());
        System.out.println("received: topic    " + data.topic());
    }
}


public class KafkaUtilTest {

    private Map<String, Object> consumerProps() {
        Map<String, Object> props = new HashMap<>();
        //  props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.1.1:9092");
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "foo");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        return props;
    }
    private KafkaMessageListenerContainer<Integer, String> createContainer(
            ContainerProperties containerProps) {
        Map<String, Object> props = consumerProps();
        DefaultKafkaConsumerFactory<Integer, String> cf =
                new DefaultKafkaConsumerFactory<Integer, String>(props);
        KafkaMessageListenerContainer<Integer, String> container =
                new KafkaMessageListenerContainer<>(cf, containerProps);
        return container;
    }

    @Test
    public void testSendMesessage()
    {
        System.out.println("start test ");
        ContainerProperties containerProps = new ContainerProperties("topic1", "topic2");
        MyListener myListener = new MyListener();
        containerProps.setMessageListener(myListener);
        KafkaMessageListenerContainer<Integer, String> container = createContainer(containerProps);
        container.setBeanName("testAuto");
        container.start();
        try {
            Thread.sleep(1000);         //此处必须等待
        }
        catch (Exception   e)
        {
            //
        }

        KafkaTemplate<Integer, String> template =  KafkaUtil.initSender("172.16.130.21:9092");
        template.send("topic1", 0, "1hello1");
        template.send("topic1",  "1hello2");
        template.send("topic1",  "1hello3");
        template.flush();
        System.out.println("end test ");
        int kk = 0;(这里用个断点，或者latch.await等待下）

        container.stop();

    }

}

最后补下pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>GroupKafka</groupId>
  <artifactId>ArtKafka</artifactId>
  <packaging>war</packaging>
  <version>1.0-SNAPSHOT</version>
  <name>ArtKafka Maven Webapp</name>
  <url>http://maven.apache.org</url>
  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.springframework.kafka</groupId>
      <artifactId>spring-kafka</artifactId>
      <version>1.2.1.RELEASE</version>
    </dependency>
    <dependency>
      <groupId>org.springframework</groupId>
      <artifactId>spring-test</artifactId>
      <version>4.3.4.RELEASE</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.springframework.kafka/spring-kafka-test -->
    <dependency>
      <groupId>org.springframework.kafka</groupId>
      <artifactId>spring-kafka-test</artifactId>
      <version>1.2.1.RELEASE</version>
    </dependency>

    <!-- log4j -->
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>1.7.5</version>
    </dependency>
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-api</artifactId>
      <version>2.7</version>
    </dependency>
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-core</artifactId>
      <version>2.7</version>
    </dependency>

  </dependencies>
  <build>
    <finalName>ArtKafka</finalName>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-resources-plugin</artifactId>
        <version>2.7</version>
        <configuration>
          <encoding>UTF-8</encoding>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>3.5</version>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
          <encoding>UTF-8</encoding>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>