Kafka API

Kafka API 地址：http://kafka.apache.org/11/javadoc/index.html?index-all.html

一、原生代码

加入依赖：

<dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka-clients</artifactId>
     <version>2.3.0</version>
</dependency>
<dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka_2.11</artifactId>
     <version>2.3.0</version>
</dependency>
<dependency>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-client</artifactId>
     <version>2.7.4</version>
</dependency>

生产者：

package csdn;
import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

public class Producer {
    private static KafkaProducer<String, String> producer;

    static {
        Properties props = new Properties();
        //声生产者kafkka地址
        props.put("bootstrap.servers", "hdp-3:9092");
        //key序列化
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        producer = new KafkaProducer<>(props);
    }

    //发送消息
    private static void send() {
        //topic   key   推送的消息
        ProducerRecord<String, String> record = new ProducerRecord<String, String>("xin", "key2", "hello wang");
        producer.send(record);
        producer.close();
    }

    public static void main(String[] args) {
        send();
    }
}

消费者：

package csdn;

import java.util.Collections;
import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;


public class Consumer {
    private static KafkaConsumer<String, String> consumer;    //接受数据方法
    private static Properties props;           //传递的参数
    static {
        props = new Properties();
        //消费者kafka地址 不用和producer地址一样
        props.put("bootstrap.servers", "hdp-2:9092");
        props.put("zookeeper.connect","hdp-1:2181,hdp-2:2181,hdp-3:2181");
        //key反序列化
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        //组???????
        props.put("group.id", "wang");
    }

    private static void ConsumerMessage() {
        HDFSWriter hdfsWriter = new HDFSWriter();
        //允许自动提交位移
        props.put("enable.auto.commit", true);
        consumer = new KafkaConsumer<String, String>(props);
        consumer.subscribe(Collections.singleton("xin"));



        //使用轮询拉取数据--消费完成之后会根据设置时长来清除消息，被消费过的消息，如果想再次被消费，可以根据偏移量(offset)来获取
        try {
            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(100);   //获取数据
                for (ConsumerRecord<String, String> r : records) {
                    System.out.printf("topic = %s, offset = %s, key = %s, value = %s", r.topic(), r.offset(),
                            r.key(), r.value());
                    hdfsWriter.writer(r.toString());
                }
            }

        } finally {
            consumer.close();
        }
    }

    public static void main(String[] args) {
        ConsumerMessage();
    }

}

 

其他详细api转载：https://blog.csdn.net/wangzhanzheng/article/details/80801059

二、SpringBoot整合Kafka

依赖

<!-- https://mvnrepository.com/artifact/org.springframework.kafka/spring-kafka -->
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
            <version>2.2.0.RELEASE</version>
        </dependency>

生产者：

package com.example.kafkaspringbt.Controller;

import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.Resource;

@RestController
@RequestMapping("/kafka")
public class Producer {
    @Resource
    private KafkaTemplate<String,String> template;

    @RequestMapping(value = {"/producer/{msg}"}, method = RequestMethod.GET)
    public String send(@PathVariable(name = "msg") String message) {
        template.send("xin",message);
        return "success";
    }
}

消费者：

package com.example.kafkaspringbt.Controller;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

@Component
public class Consumer {
    @KafkaListener(topics = "xin")
    public void accpeotMsg(ConsumerRecord<?,?> consumer) {
        System.out.printf("topic = %s, offset = %d, value = %s \n", consumer.topic(), consumer.offset(), consumer.value());
    }
}