Kafka -- 03 【主题，生产者，消费者（java/scala）的实现，自定义分区器】

1、API创建主题

1.1、java实现

package KafkaDay02;

import kafka.utils.ZkUtils;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.CreateTopicsResult;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.common.security.JaasUtils;

import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutionException;

public class _07HomeWork01 {
    public static void main(String[] args) {
        //创建一个配置对象
        Properties pro = new Properties();

        //绑定属性
        pro.setProperty("bootstrap.servers","qianfeng01:9092,qianfeng02:9092,qianfeng03:9092");

        //得到AdminClient
        AdminClient adminClient = AdminClient.create(pro);

        //获取主题集合
        List<NewTopic> topic = new ArrayList<>();

        //创建主题对象
        NewTopic newTopic = new NewTopic("test-ex1", 4, (short) 3);
        topic.add(newTopic);

        //创建主题
        CreateTopicsResult topics = adminClient.createTopics(topic);

        try {
            topics.all().get();
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        }

    }
}

1.2、scala实现

package com.xxx.Kafka.Day02

import java.util
import java.util.Properties

import org.apache.kafka.clients.admin.{AdminClient, CreateTopicsResult, NewTopic}


object _01HomeWork01 {
    def main(args: Array[String]): Unit = {
        val pro = new Properties()

        pro.put("bootstrap.servers", "qianfeng01:9092,qianfeng02:9092,qianfeng03:9092")

        val client: AdminClient = AdminClient.create(pro)

        val newTopic = new NewTopic("test-ex3", 4, 3)

        val topics = new util.ArrayList[NewTopic]()

        topics.add(newTopic)

        val result: CreateTopicsResult = client.createTopics(topics)

//        result.all().get()

        client.close()
    }
}

2、API创建生产者

2.1、java实现

package KafkaDay02;

import kafka.Kafka;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.io.IOException;
import java.util.Properties;

public class _08HomeWork2 {
    public static void main(String[] args) throws IOException {
        //创建一个配置对象
        Properties pro = new Properties();

        //绑定配置文件
        pro.load(_08HomeWork2.class.getClassLoader().getResourceAsStream("producer.properties"));

        //创建producer对象
        KafkaProducer<Integer, String> producer = new KafkaProducer<>(pro);

        //创建消息，并发送
        for (int i = 0 ; i < 10000; i++){
            ProducerRecord<Integer, String> message = new ProducerRecord<Integer, String>("test-ex1", "homework"+i);
            producer.send(message);
        }

        //关闭
        producer.close();
    }
}

2.2、scala实现

package com.xxx.Kafka.Day02

import java.util.Properties

import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}

/**
 * scala 的生产者
 */
object _02HomeWork02 {
    def main(args: Array[String]): Unit = {
        val pro = new Properties()

        pro.load(_02HomeWork02.getClass.getClassLoader.getResourceAsStream("producer.properties"))

        val producer = new KafkaProducer[Int, String](pro)

        for ( i <- 1 to 10000){
            val message = new ProducerRecord[Int, String]("test-ex1", "hello" + i)
            producer.send(message)
        }

        producer.close()
    }
}

3、API创建消费者

3.1、java实现

package KafkaDay02;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;

public class _09HomeWork03 {
    public static void main(String[] args) throws IOException {
        //配置对象
        Properties pro = new Properties();

        //绑定
        pro.load(_09HomeWork03.class.getClassLoader().getResourceAsStream("consumer.properties"));

        //创建消费者
        KafkaConsumer<Integer, String> consumer = new KafkaConsumer<>(pro);

        //绑定消费主题
        List<String> topic = new ArrayList<>();
        topic.add("test-ex1");
        consumer.subscribe(topic);

        //循环拉取消息,打印
        while(true){
            ConsumerRecords<Integer, String> message = consumer.poll(1000);
            Iterator<ConsumerRecord<Integer, String>> ite = message.iterator();

            while(ite.hasNext()){
                ConsumerRecord<Integer, String> mess = ite.next();
                System.out.println("主题"+mess.topic()+"  分区"+mess.partition()+"  值"+mess.value()+"  偏移量"+mess.offset());
            }

        }
    }
}

3.2、scala实现

package com.xxx.Kafka.Day02

import java.util
import java.util.{Collections, Properties}

import org.apache.kafka.clients.consumer.{ConsumerRecord, ConsumerRecords, KafkaConsumer}

/**
 * scala 的消费者
 */

object _03HomeWork03 {
    def main(args: Array[String]): Unit = {
        //先获取配置对象
        val pro = new Properties()

        //绑定配置文件
        pro.load(_03HomeWork03.getClass.getClassLoader.getResourceAsStream("consumer.properties"))

        //获取一个消费者
        val consumer = new KafkaConsumer[Int, String](pro)

        //订阅主题
        consumer.subscribe(Collections.singletonList("test-ex1"))

        //循环拉取
        while (true) {
            val message: ConsumerRecords[Int, String] = consumer.poll(1000)
            val ite: util.Iterator[ConsumerRecord[Int, String]] = message.iterator()
            while (ite.hasNext) {
                val messa: ConsumerRecord[Int, String] = ite.next()
                println(s"topic:${messa.topic()},partiton:${messa.partition()},value:${messa.value()},offset:${messa.offset()}")
            }
        }
    }
}

4、自定义分区器

4.1、随机分区器

package KafkaDay02;

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.PartitionInfo;

import java.util.Map;

public class _03CustomRandomPartitioner implements Partitioner {
    /**
     * 自定义随机分区器
     *
     * @param topic：主题
     * @param key：消息的key
     * @param keybytes：key的字节数组
     * @param value：消息的value
     * @param valuebytes：value的字节数组
     * @param cluster：kafka集群
     * @return：
     */
    @Override
    public int partition(String topic, Object key, byte[] keybytes, Object value, byte[] valuebytes, Cluster cluster) {
        //先获取主题中一共有几个分区
        Integer partitionCount = cluster.partitionCountForTopic(topic);
        
        //随机分区号，是0->partitionNumber-1之间的数
        int partitonNumber = (int) (Math.random() * partitionCount);

        //返回分区号
        return partitonNumber;
    }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> map) {

    }
}

4.2、Hash分区器

package KafkaDay02;

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;

import java.util.Map;

public class _04CustomHashPartitioner implements Partitioner {
    /**
     * 自定义随机分区器
     *
     * @param topic：主题
     * @param key：消息的key
     * @param keybytes：key的字节数组
     * @param value：消息的value
     * @param valuebytes：value的字节数组
     * @param cluster：kafka集群
     * @return：
     */
    @Override
    public int partition(String topic, Object key, byte[] keybytes, Object value, byte[] valuebytes, Cluster cluster) {
        //先获取主题内一共有多少分区
        Integer PartitionCount = cluster.partitionCountForTopic(topic);

        //获取key的hash值，然后和分区总数求模，获得所在的分区
        int PartitionNumber = key.hashCode() % PartitionCount;

        //返回
        return PartitionNumber;
    }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> map) {

    }
}

4.3、轮询分区器

package KafkaDay02;

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;

import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;

public class _05CustomPollingPartitioner implements Partitioner {
    /**
     * 自定义随机分区器
     *
     * @param topic：主题
     * @param key：消息的key
     * @param keybytes：key的字节数组
     * @param value：消息的value
     * @param valuebytes：value的字节数组
     * @param cluster：kafka集群
     * @return：
     */
    //先需要获取一个全局的计数器
    private AtomicInteger atomicInteger = new AtomicInteger();
    
    @Override
    public int partition(String topic, Object key, byte[] keybytes, Object value, byte[] valuebytes, Cluster cluster) {
        //先获取主题内一共有多少分区
        Integer PartitionCount = cluster.partitionCountForTopic(topic);

        //获取计数器上的值
        int andIncrement = atomicInteger.getAndIncrement();
        
        //对分区数量取模运算，就可以达到轮询效果
        int PartitionNumber = andIncrement % PartitionCount;

        //返回
        return PartitionNumber;
    }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> map) {

    }
}