Kafka使用_kafka 9094-CSDN博客

本文为博主原创文章，未经博主-撒忆雨(Ryan)允许不得转载。

本文链接：https://blog.csdn.net/u011009161/article/details/108684477

Kafka使用

Docker 集群搭建
单机手动安装版
Java 对接
面试题

Docker 集群搭建

单机手动安装版

https://www.cnblogs.com/answerThe/p/11267129.html

依赖zookeeper

见 ZooKeeper使用与原理 - Docker 安装

Docker compose

创建network
docker network create zookeeper_default --driver bridge
192.168.0.227 为本机本地IP

version: '2'

networks:
  zookeeper:
    external: true
    name: zookeeper_default

services:
  kafka1:
    image: 'bitnami/kafka:latest'
    ports:
      - '9094:9094'
    container_name: kafka1
    networks:
      - zookeeper
    environment:
      - KAFKA_ZOOKEEPER_CONNECT=zookeeperHostIp:2181,zookeeperHostIp:2182,zookeeperHostIp:2183/kafka
      - KAFKA_BROKER_ID=1
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9094
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://KafkaHostIp:9094
    volumes:
      - /opt/kafka/kafka1/persistence:/bitnami/kafka
      - /etc/localtime:/etc/localtime

  kafka2:
    image: 'bitnami/kafka:latest'
    ports:
      - '9092:9092'
    container_name: kafka2
    networks:
      - zookeeper
    environment:
      - KAFKA_ZOOKEEPER_CONNECT=zookeeperHostIp:2181,zookeeperHostIp:2182,zookeeperHostIp:2183/kafka
      - KAFKA_BROKER_ID=2
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://KafkaHostIp:9092
    volumes:
      - /opt/kafka/kafka2/persistence:/bitnami/kafka
      - /etc/localtime:/etc/localtime

  kafka3:
    image: 'bitnami/kafka:latest'
    ports:
      - '9093:9093'
    container_name: kafka3
    networks:
      - zookeeper
    environment:
      - KAFKA_ZOOKEEPER_CONNECT=zookeeperHostIp:2181,zookeeperHostIp:2182,zookeeperHostIp:2183/kafka
      - KAFKA_BROKER_ID=3
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9093
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://KafkaHostIp:9093
    volumes:
      - /opt/kafka/kafka3/persistence:/bitnami/kafka
      - /etc/localtime:/etc/localtime
      
  kafka-manager:
    image: sheepkiller/kafka-manager:1.2.7
    ports:
      - '9000:9000'
    environment:
      - ZK_HOSTS=zookeeperHostIp:2181,zookeeperHostIp:2182,zookeeperHostIp:2183/kafka
      - APPLICATION_SECRET=ryan123
    volumes:
      - /opt/kafka-admin/confdir:/kafka-manager-1.2.7/conf
      - /etc/localtime:/etc/localtime
    networks:
      - zookeeper

docker-compose up 汇报没有application.conf 配置文件

# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
# See accompanying LICENSE file.

# This is the main configuration file for the application.
# ~~~~~

# Secret key
# ~~~~~
# The secret key is used to secure cryptographics functions.
# If you deploy your application to several instances be sure to use the same key!
play.crypto.secret="^<csmm5Fx4d=r2HEX8pelM3iBkFVv?k[mc;IZE<_Qoq8EkX_/7@Zt6dP05Pzea3U"
play.crypto.secret=${?APPLICATION_SECRET}

# The application languages
# ~~~~~
play.i18n.langs=["en"]

play.http.requestHandler = "play.http.DefaultHttpRequestHandler"
play.http.context = "/"
play.application.loader=loader.KafkaManagerLoader

kafka-manager.zkhosts="kafka-manager-zookeeper:2181"
kafka-manager.zkhosts=${?ZK_HOSTS}
pinned-dispatcher.type="PinnedDispatcher"
pinned-dispatcher.executor="thread-pool-executor"
application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"]

akka {
  loggers = ["akka.event.slf4j.Slf4jLogger"]
  loglevel = "INFO"
}


basicAuthentication.enabled=false
basicAuthentication.username="admin"
basicAuthentication.password="password"
basicAuthentication.realm="Kafka-Manager"


kafka-manager.consumer.properties.file=${?CONSUMER_PROPERTIES_FILE}

验证

创建Topic

./kafka-topics.sh --create --bootstrap-server 192.168.16.1:9092,192.168.16.1:9093,192.168.16.1:9094 --topic test123   --replica-assignment 0:1,1:2,2:0

消费监听

./kafka-console-consumer.sh --bootstrap-server 192.168.16.1:9092,192.168.16.1:9093,192.168.16.1:9094 --topic test123 --from-beginning

发布消息

./kafka-console-producer.sh --broker-list 192.168.16.1:9092,192.168.16.1:9093,192.168.16.1:9094 --topic test123

分组查看

 ./kafka-consumer-groups.sh --bootstrap-server 192.168.16.1:9092,192.168.16.1:9093,192.168.16.1:9094 --all-groups --list --all-topics

Kafka Manager

Java 对接

POM引入

<dependency>
   <groupId>org.apache.kafka</groupId>
   <artifactId>kafka_2.10</artifactId>
   <version>0.8.2.0</version>
 </dependency>

消费者

package org.example.kafka;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

/**
 * @Auther: Ryan
 * @Date: 2022/1/21 14:05
 * @Description: 接收数据
 */
public class kafkaConsumer extends Thread{

    private final String topic;

    public kafkaConsumer(String topic){
        super();
        this.topic = topic;
    }

    @Override
    public void run() {
        ConsumerConnector consumer = createConsumer();
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, 1); // 一次从主题中获取一个数据
        Map<String, List<KafkaStream<byte[], byte[]>>>  messageStreams = consumer.createMessageStreams(topicCountMap);
        KafkaStream<byte[], byte[]> stream = messageStreams.get(topic).get(0);// 获取每次接收到的这个数据
        for (kafka.message.MessageAndMetadata<byte[], byte[]> messageAndMetadata : stream) {
            String message = new String(messageAndMetadata.message());
            System.out.println("接收到: " + message);
        }
    }

    private ConsumerConnector createConsumer() {
        Properties properties = new Properties();
        properties.put("zookeeper.connect", "192.168.16.1:2181,192.168.16.1:2182,192.168.16.1:2183");//声明zk
        properties.put("group.id", "group1");
        return Consumer.createJavaConsumerConnector(new ConsumerConfig(properties));
    }

    public static void main(String[] args) {
        new kafkaConsumer("test123").start();// 使用kafka集群中创建好的主题 test123
    }

}

生产者

package org.example.kafka;

import java.util.Properties;
import java.util.concurrent.TimeUnit;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import kafka.serializer.StringEncoder;

/**
 * @Auther: Ryan
 * @Date: 2022/1/21 14:05
 * @Description: 发送Kafka消息
 */
public class kafkaProducer extends Thread{

    private final String topic;

    public kafkaProducer(String topic){
        super();
        this.topic = topic;
    }

    @Override
    public void run() {
        Producer<Integer, String> producer = createProducer();
        int i=0;
        while(true){
            producer.send(new KeyedMessage<Integer, String>(topic, "message: " + i++));
            try {
                TimeUnit.SECONDS.sleep(1);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    private Producer<Integer, String> createProducer() {
        Properties properties = new Properties();
        properties.put("zookeeper.connect", "192.168.16.1:2181,192.168.16.1:2182,192.168.16.1:2183");//声明zk
        properties.put("serializer.class", StringEncoder.class.getName());
        properties.put("metadata.broker.list", "192.168.16.1:9092,192.168.16.1:9093,192.168.16.1:9094");// 声明kafka broker
        return new Producer<Integer, String>(new ProducerConfig(properties));
    }
    
    public static void main(String[] args) {
        new kafkaProducer("test123").start();// 使用kafka集群中创建好的主题 test123
    }
}