Kafka low level API8 Consumer

最新推荐文章于 2021-01-27 15:09:00 发布
最新推荐文章于 2021-01-27 15:09:00 发布
阅读量728 收藏
点赞数
分类专栏： Kafka 文章标签： bigdata kafka
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/wingofeagle/article/details/61192330
版权 
  
  

   
   

    
    import
    
     java
    
    .
    
    nio
    
    .
    
    ByteBuffer
    
    ;

   
   

   
   

    
    import java.util.ArrayList;
   
   

   
   

    
    import java.util.Collections;
   
   

   
   

    
    import java.util.HashMap;
   
   

   
   

    
    import java.util.List;
   
   

   
   

    
    import java.util.Map;
   
   

   
   

    
     
   
   

   
   

    
    import org.apache.log4j.Logger;
   
   

   
   

    
     
   
   

   
   

    
    import com.jd.Seckill.UdpRecv.Utils.Config;
   
   

   
   

    
     
   
   

   
   

    
    import kafka.api.FetchRequest;
   
   

   
   

    
    import kafka.api.FetchRequestBuilder;
   
   

   
   

    
    import kafka.api.PartitionOffsetRequestInfo;
   
   

   
   

    
    import kafka.cluster.Broker;
   
   

   
   

    
    import kafka.common.ErrorMapping;
   
   

   
   

    
    import kafka.common.TopicAndPartition;
   
   

   
   

    
    import kafka.javaapi.FetchResponse;
   
   

   
   

    
    import kafka.javaapi.OffsetRequest;
   
   

   
   

    
    import kafka.javaapi.OffsetResponse;
   
   

   
   

    
    import kafka.javaapi.PartitionMetadata;
   
   

   
   

    
    import kafka.javaapi.TopicMetadata;
   
   

   
   

    
    import kafka.javaapi.TopicMetadataRequest;
   
   

   
   

    
    import kafka.javaapi.consumer.SimpleConsumer;
   
   

   
   

    
    import kafka.message.MessageAndOffset;
   
   

   
   

    
    import kafka.javaapi.TopicMetadataResponse;
   
   

   
   

    
     
   
   

   
   

    
    publicclassKafkaConsume_LowAPI8
   
   

   
   

    
    {
   
   

   
   

    
    privatestaticLogger log =Logger.getLogger(KafkaConsume_LowAPI8.class);
   
   

   
   

    
    privateList<String> m_replicaBrokers =newArrayList<String>();
   
   

   
   

    
     
   
   

   
   

    
    publicKafkaConsume_LowAPI8()
   
   

   
   

    
    {
   
   

   
   

    
    		m_replicaBrokers =newArrayList<String>();
   
   

   
   

    
    }
   
   

   
   

    
     
   
   

   
   

    
    publicvoid run(long a_maxReads,String a_topic,int a_partition,List<String> a_seedBrokers,int a_port)throwsException
   
   

   
   

    
    {
   
   

   
   

    
    // 找出目的topic与分区的leader broker（负责数据的读取与写入）
   
   

   
   

    
    PartitionMetadata metadata = findLeader(a_seedBrokers, a_port, a_topic, a_partition);
   
   

   
   

    
    if(null== metadata)
   
   

   
   

    
    {
   
   

   
   

    
    			log.info("Can't find metadata for Topic and Partition. Exiting");
   
   

   
   

    
    return;
   
   

   
   

    
    }
   
   

   
   

    
    if(null== metadata.leader())
   
   

   
   

    
    {
   
   

   
   

    
    			log.info("Can't find Leader for Topic and Partition. Exiting");
   
   

   
   

    
    return;
   
   

   
   

    
    }
   
   

   
   

    
    String leadBroker = metadata.leader().host();
   
   

   
   

    
    String clientName ="Client_"+ a_topic +"_"+ a_partition;
   
   

   
   

    
     
   
   

   
   

    
    SimpleConsumer consumer =newSimpleConsumer(leadBroker, a_port,100000,64*1024, clientName);
   
   

   
   

    
    // 获取要读取的offset数值
   
   

   
   

    
    long readOffset = getLastOffset(consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
   
   

   
   

    
     
   
   

   
   

    
    int numErrors =0;
   
   

   
   

    
    while(a_maxReads >0)
   
   

   
   

    
    {
   
   

   
   

    
    if(consumer ==null)
   
   

   
   

    
    				consumer =newSimpleConsumer(leadBroker, a_port,100000,64*1024, clientName);
   
   

   
   

    
     
   
   

   
   

    
    // Reading the Data,获取数据
   
   

   
   

    
    FetchRequest req =newFetchRequestBuilder().clientId(clientName).addFetch(a_topic, a_partition, readOffset,100000).build();
   
   

   
   

    
    FetchResponse fetchResponse = consumer.fetch(req);
   
   

   
   

    
     
   
   

   
   

    
    if(fetchResponse.hasError())
   
   

   
   

    
    {
   
   

   
   

    
    				numErrors++;
   
   

   
   

    
    // Something went wrong!
   
   

   
   

    
    short code = fetchResponse.errorCode(a_topic, a_partition);
   
   

   
   

    
    				log.error("Error fetching data from the Broker:"+ leadBroker +" Reason: "+ code);
   
   

   
   

    
    if(numErrors >5)
   
   

   
   

    
    break;
   
   

   
   

    
    if(code ==ErrorMapping.OffsetOutOfRangeCode())
   
   

   
   

    
    {
   
   

   
   

    
    // We asked for an invalid offset. For simple case ask for
   
   

   
   

    
    // the last element to reset
   
   

   
   

    
    					readOffset = getLastOffset(consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
   
   

   
   

    
    continue;
   
   

   
   

    
    }
   
   

   
   

    
    				consumer.close();
   
   

   
   

    
    				consumer =null;
   
   

   
   

    
    				leadBroker = findNewLeader(leadBroker, a_topic, a_partition, a_port);
   
   

   
   

    
    continue;
   
   

   
   

    
    }
   
   

   
   

    
    			numErrors =0;
   
   

   
   

    
     
   
   

   
   

    
    long numRead =0;
   
   

   
   

    
    for(MessageAndOffset messageAndOffset : fetchResponse.messageSet(a_topic, a_partition))
   
   

   
   

    
    {
   
   

   
   

    
    long currentOffset = messageAndOffset.offset();
   
   

   
   

    
    if(currentOffset < readOffset)
   
   

   
   

    
    {
   
   

   
   

    
    					log.error("Found an old offset: "+ currentOffset +" Expecting: "+ readOffset);
   
   

   
   

    
    continue;
   
   

   
   

    
    }
   
   

   
   

    
    				readOffset = messageAndOffset.nextOffset();
   
   

   
   

    
    ByteBuffer payload = messageAndOffset.message().payload();
   
   

   
   

    
    byte[] bytes =newbyte[payload.limit()];
   
   

   
   

    
    				payload.get(bytes);
   
   

   
   

    
    				log.info(String.valueOf(messageAndOffset.offset())+": "+newString(bytes,"UTF-8"));
   
   

   
   

    
    				numRead++;
   
   

   
   

    
    				a_maxReads--;
   
   

   
   

    
    }
   
   

   
   

    
     
   
   

   
   

    
    if(numRead ==0)
   
   

   
   

    
    {
   
   

   
   

    
    try
   
   

   
   

    
    {
   
   

   
   

    
    Thread.sleep(1000);
   
   

   
   

    
    }
   
   

   
   

    
    catch(InterruptedException ie)
   
   

   
   

    
    {
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    if(consumer !=null)
   
   

   
   

    
    			consumer.close();
   
   

   
   

    
    }
   
   

   
   

    
     
   
   

   
   

    
    // Now define where to start reading data. Kafka includes two constants to
   
   

   
   

    
    // help, kafka.api.OffsetRequest.EarliestTime() finds the beginning of the
   
   

   
   

    
    // data in the logs and starts streaming from there,
   
   

   
   

    
    // kafka.api.OffsetRequest.LatestTime() will only stream new messages. Don’t
   
   

   
   

    
    // assume that offset 0 is the beginning offset, since messages age out of
   
   

   
   

    
    // the log over time
   
   

   
   

    
    publicstaticlong getLastOffset(SimpleConsumer consumer,String topic,int partition,long whichTime,String clientName)
   
   

   
   

    
    {
   
   

   
   

    
    		log.info("getLastOffset,topic:"+ topic +",partition:"+ partition +",whichTime:"+ whichTime +",clientName:"+ clientName);
   
   

   
   

    
    TopicAndPartition topicAndPartition =newTopicAndPartition(topic, partition);
   
   

   
   

    
    Map<TopicAndPartition,PartitionOffsetRequestInfo> requestInfo =newHashMap<TopicAndPartition,PartitionOffsetRequestInfo>();
   
   

   
   

    
    		requestInfo.put(topicAndPartition,newPartitionOffsetRequestInfo(whichTime,1));
   
   

   
   

    
    OffsetRequest request =newOffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
   
   

   
   

    
    OffsetResponse response = consumer.getOffsetsBefore(request);
   
   

   
   

    
    if(response.hasError())
   
   

   
   

    
    {
   
   

   
   

    
    System.out.println("Error fetching data Offset Data the Broker. Reason: "+ response.errorCode(topic, partition));
   
   

   
   

    
    return0;
   
   

   
   

    
    }
   
   

   
   

    
    long[] offsets = response.offsets(topic, partition);
   
   

   
   

    
    return offsets[0];
   
   

   
   

    
    }
   
   

   
   

    
     
   
   

   
   

    
    privateString findNewLeader(String a_oldLeader,String a_topic,int a_partition,int a_port)throwsException
   
   

   
   

    
    {
   
   

   
   

    
    for(int i =0; i <3; i++)
   
   

   
   

    
    {
   
   

   
   

    
    boolean goToSleep =false;
   
   

   
   

    
    PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
   
   

   
   

    
    if(metadata ==null)
   
   

   
   

    
    {
   
   

   
   

    
    				goToSleep =true;
   
   

   
   

    
    }
   
   

   
   

    
    elseif(metadata.leader()==null)
   
   

   
   

    
    {
   
   

   
   

    
    				goToSleep =true;
   
   

   
   

    
    }
   
   

   
   

    
    elseif(a_oldLeader.equalsIgnoreCase(metadata.leader().host())&& i ==0)
   
   

   
   

    
    {
   
   

   
   

    
    				goToSleep =true;
   
   

   
   

    
    }
   
   

   
   

    
    else
   
   

   
   

    
    {
   
   

   
   

    
    return metadata.leader().host();
   
   

   
   

    
    }
   
   

   
   

    
    if(goToSleep)
   
   

   
   

    
    {
   
   

   
   

    
    try
   
   

   
   

    
    {
   
   

   
   

    
    Thread.sleep(1000);
   
   

   
   

    
    }
   
   

   
   

    
    catch(InterruptedException e)
   
   

   
   

    
    {
   
   

   
   

    
    					log.error(e);
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    		log.error("Unable to find new leader after Broker failure. Exiting");
   
   

   
   

    
    thrownewException("Unable to find new leader after Broker failure. Exiting");
   
   

   
   

    
    }
   
   

   
   

    
     
   
   

   
   

    
    // 获取指定topic与分区的broker leader
   
   

   
   

    
    privatePartitionMetadata findLeader(List<String> a_seedBrokers,int a_port,String a_topic,int a_partition)
   
   

   
   

    
    {
   
   

   
   

    
    PartitionMetadata returnMetaData =null;
   
   

   
   

    
     
   
   

   
   

    
    		loop:for(String seed : a_seedBrokers)
   
   

   
   

    
    {
   
   

   
   

    
    SimpleConsumer consumer =null;
   
   

   
   

    
    try
   
   

   
   

    
    {
   
   

   
   

    
    // soTimeout与buffersize是什么意思？
   
   

   
   

    
    				consumer =newSimpleConsumer(seed, a_port,100000,64*1024,"leaderLookup");
   
   

   
   

    
    // singletonlist返回一个只包含指定对象的不可变列表
   
   

   
   

    
    List<String> topics =Collections.singletonList(a_topic);
   
   

   
   

    
    TopicMetadataRequest req =newTopicMetadataRequest(topics);
   
   

   
   

    
    TopicMetadataResponse resp = consumer.send(req);
   
   

   
   

    
     
   
   

   
   

    
    List<TopicMetadata> metaData = resp.topicsMetadata();
   
   

   
   

    
    for(TopicMetadata item : metaData)
   
   

   
   

    
    {
   
   

   
   

    
    for(PartitionMetadata part : item.partitionsMetadata())
   
   

   
   

    
    {
   
   

   
   

    
    if(part.partitionId()== a_partition)
   
   

   
   

    
    {
   
   

   
   

    
    							returnMetaData = part;
   
   

   
   

    
    // 跳出外循环；如果只break那么只会跳出内循环
   
   

   
   

    
    break loop;
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    catch(Exception e)
   
   

   
   

    
    {
   
   

   
   

    
    System.out.println(
   
   

   
   

    
    "Error communicating with Broker ["+ seed +"] to find Leader for ["+ a_topic +", "+ a_partition +"] Reason: "+ e);
   
   

   
   

    
    }
   
   

   
   

    
    finally
   
   

   
   

    
    {
   
   

   
   

    
    if(consumer !=null)
   
   

   
   

    
    					consumer.close();
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    if(returnMetaData !=null)
   
   

   
   

    
    {
   
   

   
   

    
    			m_replicaBrokers.clear();
   
   

   
   

    
    for(Broker replica : returnMetaData.replicas())
   
   

   
   

    
    {
   
   

   
   

    
    				m_replicaBrokers.add(replica.host());
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    return returnMetaData;
   
   

   
   

    
    }
   
   

   
   

    
     
   
   

   
   

    
    publicvoid mainHandle()
   
   

   
   

    
    {
   
   

   
   

    
    KafkaConsume_LowAPI8 example =newKafkaConsume_LowAPI8();
   
   

   
   

    
    long maxReads =25;
   
   

   
   

    
    String topic =Config.getM_strConsume8Topic();
   
   

   
   

    
    int partition =0;
   
   

   
   

    
    List<String> seeds =newArrayList<String>();
   
   

   
   

    
    		seeds.add("10.189.122.213");
   
   

   
   

    
    		seeds.add("10.189.122.207");
   
   

   
   

    
    		seeds.add("10.189.122.208");
   
   

   
   

    
    int port =9092;
   
   

   
   

    
    try
   
   

   
   

    
    {
   
   

   
   

    
    			example.run(maxReads, topic, partition, seeds, port);
   
   

   
   

    
    }
   
   

   
   

    
    catch(Exception e)
   
   

   
   

    
    {
   
   

   
   

    
    System.out.println("Oops:"+ e);
   
   

   
   

    
    			e.printStackTrace();
   
   

   
   

    
    }
   
   

   
   

    
    }
   
   

   
   

    
    }

Downsides of using SimpleConsumer

The SimpleConsumer does require a significant amount of work not needed in the Consumer Groups:

  1.           You must keep track of the offsets in your application to know where you left off consuming.
  2.      You must figure out which Broker is the lead Broker for a topic and partition
  3.      You  must handle Broker leader changes

Steps for using a SimpleConsumer

  1. .    Find an active Broker and find out which Broker is the leader for your topic and partition
  2.      Determine who the replica Brokers are for your topic and partition
  3.     Build the request defining what data you are interested in
  4.     .Fetch the data
  5.      Identify and recover from leader changes
_北方的雪_
关注
专栏目录
Kafka高级(2) -- Kafka高级（High Level）API与低级（Low Level）API
erainm
09-16 957
1. 高级API 2. 低级API 3. 手动消费分区数据
Kafka 实现之接口设计 —— 生产者 API、消费者 API
最新发布
流华追梦的专栏
03-13 1075
Kafka 强大的应用程序层的基础是用于访问存储的两个基本 API，即用于写入事件的生产者 API和用于读取事件的消费者 API。在这两个 API 之上构建了用于集成和处理的 API。
kafka-生产者，消费者 lowlevel api
strideahead的博客
08-08 365
import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; import kafka.api.PartitionOffsetRequestInfo; import kafka.cluster.BrokerEndPoint; import kafka.common.ErrorMapping; import kafka....
kafka 低级api_使用低级API分5步在sagemaker中建立模型
weixin_26716079的博客
10-09 230
kafka 低级apiIn this article, we will build a model using low-level API in AWS SageMaker. If you are a beginner, start with the post for high-level API (where steps one and two of this guide are explain...
Scala版SparkStreaming读写kafka，low level api模板代码存档
zzyzzy12
10-08 3952
spark streaming从kafka的某个topic拉取数据，处理完后再放入某个topic中的一个模板，不完整，作为参考，加了部分备注。     SparkStreaming主程序模板 package jacob.sparkstreaming import java.util import _root_.kafka.serializer.StringDecoder
高级 Consumer API 和低级 Consumer API
weixin_30716725的博客
08-06 92
高级 API优点 高级 API 写起来简单 不需要自行去管理 offset，系统通过 zookeeper 自行管理 不需要管理分区，副本等情况，系统自动管理 消费者断线会自动根据上一次记录在 zookeeper 中的 offset 去接着获取数据（默认设置1 分钟更新一下 zookeeper 中存的 offset） 可以使用 gr...
Kafka HighLevel API与LowLevel API说明
weixin_42005602的博客
01-27 1374
Kafka的Consumer API有两种 High Level Consumer API Low Level Consumer API 对于开发这来说，选用哪个，首先要弄清楚这两种API的工作原理什么样，分别开放了哪些功能。 High Level Consumer API说明 High Level Consumer API围绕着Consumer Group这个逻辑概念展开。 它自动管理每个Topic的每个Partition的Offset（自动读取zookeeper中该Consumer group的las
Kafka:High level consumer vs. Low level consumer
WangQYoho的博客
10-26 4214
Kafka中的消费者有两套API，分别是high level的和low level的。两种消费方式在构造和实现上都是不同的，在此记录一下： 一、High level consumer API High Level Consumer API围绕着Consumer Group这个逻辑概念展开，它屏蔽了每个Topic的每个Partition的Offset管理（自动读取zookeeper中该Consu
Kafka_API_文档
04-04
- **11.2.2 Low-Level Processor API** - **11.2.3 High-Level Streams DSL** #### 十二、安全迁移 **12.1 新集群** 描述了如何在一个全新的环境中设置Kafka集群。 **12.2 迁移现有集群** 提供了如何将现有集群...
kafka 分组消费topic_Kafka消费组(consumer group)(转）
weixin_42494742的博客
12-24 1920
在开始之前，我想花一点时间先来明确一些概念和术语，这会极大地方便我们下面的讨论。另外请原谅这文章有点长，毕竟要讨论的东西很多，虽然已然删除了很多太过细节的东西。一、 误区澄清与概念明确1 Kafka的版本很多人在Kafka中国社区(替群主做个宣传，QQ号：162272557)提问时的开头经常是这样的：“我使用的kafka版本是2.10/2.11, 现在碰到一个奇怪的问题。。。。” 无意冒犯，但这里...
记一次kafka的high level和low level
hell_world!
10-10 3960
php  rdkafka 之前的业务用的rdkafka的low level进行消息消费，没什么问题 今天一个新业务，申请的新的broker和topic，结果low level就消费不到数据了 后来改成了high level（https://arnaud-lb.github.io/php-rdkafka/phpdoc/rdkafka.examples-high-level-consumer.h...
Kafka中produer发送消息回调超时错误
weixin_33769125的博客
02-13 1598
2019独角兽企业重金招聘Python工程师标准>>> ...
【大数据kafka之low-level consumer】
06-02 161
  一、什么时候用这个接口?      1）Read a message multiple times      2）Consume only a subset of the partitions in a topic in a process     3）Manage transactions to make sure a message is processed once and ...
kafka消费者低级实例
大JAVA解决方案
05-10 837
kafka消费者低级实例半兽人 发表于: 2015-03-03   最后更新时间: 2016-10-25  订阅   1 订阅，6994 游览history教程 问答区 0kafka消费者低级实例为什么使用SimpleConsumer?使用“SimpleConsumer”的主要原因是你想比使用“消费者分组”更好的控制分区消费。比如你想:多次读取消息在一个处理过程中只消费Partition其中的一部...
Kafka学习笔记-Java简单操作
weixin_34216196的博客
08-10 93
Maven依赖包： [plain]view plaincopy <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> ...
Kafka深入 - 消费接口分析
Jason的专栏
12-30 724
1.概述 　　在 Kafka 中，官方对外提供了两种消费 API，一种是高等级消费 API，另一种是低等级的消费 API。在 《高级消费 API》一文中，介绍了其高级消费的 API 实现。今天给大家介绍另一种消费 API。 2.内容 　　在使用过 Kafka 的高级消费 API 后，我们知道它是一种高度抽象的消费 API，使用起来简单，方便，但是对于某些特殊的需求我们可能要用到第二种更加底层
Kafka(四) Kafka API实战
奥迪的博客
11-12 207
第4章 Kafka API实战 4.1 环境准备 1）启动zk和kafka集群，在kafka集群中打开一个消费者 [atguigu@node102 kafka]$ bin/kafka-console-consumer.sh \ --zookeeper node102:2181 --topic first 2）导入pom依赖 <dependencies> <!-- https://mvnrepository.com/artifact/org.apache.kafka/k...
Kafka Consumer的两种接口（高阶低阶）
liangzelei的博客
07-25 4647
Kafka Consumer接口 对于kafka的consumer接口，提供两种版本，   high-level 一种high-level版本，比较简单不用关心offset, 会自动的读zookeeper中该Consumer group的last offset 参考，https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Gro...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值