import java . nio . ByteBuffer ;import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import com.jd.Seckill.UdpRecv.Utils.Config;
import kafka.api.FetchRequest;
import kafka.api.FetchRequestBuilder;
import kafka.api.PartitionOffsetRequestInfo;
import kafka.cluster.Broker;
import kafka.common.ErrorMapping;
import kafka.common.TopicAndPartition;
import kafka.javaapi.FetchResponse;
import kafka.javaapi.OffsetRequest;
import kafka.javaapi.OffsetResponse;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataRequest;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.message.MessageAndOffset;
import kafka.javaapi.TopicMetadataResponse;
publicclassKafkaConsume_LowAPI8
{
privatestaticLogger log =Logger.getLogger(KafkaConsume_LowAPI8.class);
privateList<String> m_replicaBrokers =newArrayList<String>();
publicKafkaConsume_LowAPI8()
{
m_replicaBrokers =newArrayList<String>();
}
publicvoid run(long a_maxReads,String a_topic,int a_partition,List<String> a_seedBrokers,int a_port)throwsException
{
// 找出目的topic与分区的leader broker(负责数据的读取与写入)
PartitionMetadata metadata = findLeader(a_seedBrokers, a_port, a_topic, a_partition);
if(null== metadata)
{
log.info("Can't find metadata for Topic and Partition. Exiting");
return;
}
if(null== metadata.leader())
{
log.info("Can't find Leader for Topic and Partition. Exiting");
return;
}
String leadBroker = metadata.leader().host();
String clientName ="Client_"+ a_topic +"_"+ a_partition;
SimpleConsumer consumer =newSimpleConsumer(leadBroker, a_port,100000,64*1024, clientName);
// 获取要读取的offset数值
long readOffset = getLastOffset(consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
int numErrors =0;
while(a_maxReads >0)
{
if(consumer ==null)
consumer =newSimpleConsumer(leadBroker, a_port,100000,64*1024, clientName);
// Reading the Data,获取数据
FetchRequest req =newFetchRequestBuilder().clientId(clientName).addFetch(a_topic, a_partition, readOffset,100000).build();
FetchResponse fetchResponse = consumer.fetch(req);
if(fetchResponse.hasError())
{
numErrors++;
// Something went wrong!
short code = fetchResponse.errorCode(a_topic, a_partition);
log.error("Error fetching data from the Broker:"+ leadBroker +" Reason: "+ code);
if(numErrors >5)
break;
if(code ==ErrorMapping.OffsetOutOfRangeCode())
{
// We asked for an invalid offset. For simple case ask for
// the last element to reset
readOffset = getLastOffset(consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
continue;
}
consumer.close();
consumer =null;
leadBroker = findNewLeader(leadBroker, a_topic, a_partition, a_port);
continue;
}
numErrors =0;
long numRead =0;
for(MessageAndOffset messageAndOffset : fetchResponse.messageSet(a_topic, a_partition))
{
long currentOffset = messageAndOffset.offset();
if(currentOffset < readOffset)
{
log.error("Found an old offset: "+ currentOffset +" Expecting: "+ readOffset);
continue;
}
readOffset = messageAndOffset.nextOffset();
ByteBuffer payload = messageAndOffset.message().payload();
byte[] bytes =newbyte[payload.limit()];
payload.get(bytes);
log.info(String.valueOf(messageAndOffset.offset())+": "+newString(bytes,"UTF-8"));
numRead++;
a_maxReads--;
}
if(numRead ==0)
{
try
{
Thread.sleep(1000);
}
catch(InterruptedException ie)
{
}
}
}
if(consumer !=null)
consumer.close();
}
// Now define where to start reading data. Kafka includes two constants to
// help, kafka.api.OffsetRequest.EarliestTime() finds the beginning of the
// data in the logs and starts streaming from there,
// kafka.api.OffsetRequest.LatestTime() will only stream new messages. Don’t
// assume that offset 0 is the beginning offset, since messages age out of
// the log over time
publicstaticlong getLastOffset(SimpleConsumer consumer,String topic,int partition,long whichTime,String clientName)
{
log.info("getLastOffset,topic:"+ topic +",partition:"+ partition +",whichTime:"+ whichTime +",clientName:"+ clientName);
TopicAndPartition topicAndPartition =newTopicAndPartition(topic, partition);
Map<TopicAndPartition,PartitionOffsetRequestInfo> requestInfo =newHashMap<TopicAndPartition,PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition,newPartitionOffsetRequestInfo(whichTime,1));
OffsetRequest request =newOffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if(response.hasError())
{
System.out.println("Error fetching data Offset Data the Broker. Reason: "+ response.errorCode(topic, partition));
return0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
privateString findNewLeader(String a_oldLeader,String a_topic,int a_partition,int a_port)throwsException
{
for(int i =0; i <3; i++)
{
boolean goToSleep =false;
PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
if(metadata ==null)
{
goToSleep =true;
}
elseif(metadata.leader()==null)
{
goToSleep =true;
}
elseif(a_oldLeader.equalsIgnoreCase(metadata.leader().host())&& i ==0)
{
goToSleep =true;
}
else
{
return metadata.leader().host();
}
if(goToSleep)
{
try
{
Thread.sleep(1000);
}
catch(InterruptedException e)
{
log.error(e);
}
}
}
log.error("Unable to find new leader after Broker failure. Exiting");
thrownewException("Unable to find new leader after Broker failure. Exiting");
}
// 获取指定topic与分区的broker leader
privatePartitionMetadata findLeader(List<String> a_seedBrokers,int a_port,String a_topic,int a_partition)
{
PartitionMetadata returnMetaData =null;
loop:for(String seed : a_seedBrokers)
{
SimpleConsumer consumer =null;
try
{
// soTimeout与buffersize是什么意思?
consumer =newSimpleConsumer(seed, a_port,100000,64*1024,"leaderLookup");
// singletonlist返回一个只包含指定对象的不可变列表
List<String> topics =Collections.singletonList(a_topic);
TopicMetadataRequest req =newTopicMetadataRequest(topics);
TopicMetadataResponse resp = consumer.send(req);
List<TopicMetadata> metaData = resp.topicsMetadata();
for(TopicMetadata item : metaData)
{
for(PartitionMetadata part : item.partitionsMetadata())
{
if(part.partitionId()== a_partition)
{
returnMetaData = part;
// 跳出外循环;如果只break那么只会跳出内循环
break loop;
}
}
}
}
catch(Exception e)
{
System.out.println(
"Error communicating with Broker ["+ seed +"] to find Leader for ["+ a_topic +", "+ a_partition +"] Reason: "+ e);
}
finally
{
if(consumer !=null)
consumer.close();
}
}
if(returnMetaData !=null)
{
m_replicaBrokers.clear();
for(Broker replica : returnMetaData.replicas())
{
m_replicaBrokers.add(replica.host());
}
}
return returnMetaData;
}
publicvoid mainHandle()
{
KafkaConsume_LowAPI8 example =newKafkaConsume_LowAPI8();
long maxReads =25;
String topic =Config.getM_strConsume8Topic();
int partition =0;
List<String> seeds =newArrayList<String>();
seeds.add("10.189.122.213");
seeds.add("10.189.122.207");
seeds.add("10.189.122.208");
int port =9092;
try
{
example.run(maxReads, topic, partition, seeds, port);
}
catch(Exception e)
{
System.out.println("Oops:"+ e);
e.printStackTrace();
}
}
}
Downsides of using SimpleConsumer
The SimpleConsumer does require a significant amount of work not needed in the Consumer Groups:
- You must keep track of the offsets in your application to know where you left off consuming.
- You must figure out which Broker is the lead Broker for a topic and partition
- You must handle Broker leader changes
Steps for using a SimpleConsumer
- . Find an active Broker and find out which Broker is the leader for your topic and partition
- Determine who the replica Brokers are for your topic and partition
- Build the request defining what data you are interested in
- .Fetch the data
- Identify and recover from leader changes