Kafka组件研究<四>----Kafka API使用

4.1 生产者Demo

 

package com.kafka.test;

import java.util.HashMap;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;


/**
 * KafkaProducer
 * @Description：Kafka 生产者
 * @author 
 */
public class TestKafkaProducer extends Thread
{
	private static final Logger LOG = Logger.getLogger(TestKafkaProducer.class);
	
	/**
	 * 日志信息配置文件
	 */
	public static final String LOG4J_PROPERTIES= "conf/log4j.properties";

	static
	{
		/**
		 * 设置log信息配置文件
		 */
		PropertyConfigurator.configure(LOG4J_PROPERTIES);
	}
	
	/**
	 * 主题
	 */
	private String topic;

	/**
	 * 获取主题
	 * @return
	 */
	public String getTopic()
	{
		return topic;
	}

	/**
	 * 构造函数
	 * @param topic
	 */
	public TestKafkaProducer(String topic)
	{
		this.topic = topic;
	}

	@SuppressWarnings({ "rawtypes", "unchecked" })
	@Override
	public void run()
	{
		try
		{
			KafkaProducer producer = createProducer();
			int i = 0;
			while (true)
			{
				producer.send(new ProducerRecord<Integer, String>(topic, "message: "+ i++));
			    TimeUnit.SECONDS.sleep(1);
			}
		}
		catch (KafkaException e)
		{
			LOG.error("fail"+e,e);
		}
		catch (Exception e)
		{
			LOG.error("fail"+e,e);
		}
	
	}

	/**
	 * 获取生产者进程
	 * @return
	 */
	@SuppressWarnings({ "unchecked", "rawtypes" })
	public KafkaProducer createProducer()
	{
		HashMap<String,String> map = new HashMap<String,String>();
		/**
		 * 声明zk
		 */
		map.put("zookeeper.connect", "10.1.112.222:2181");
		map.put("zookeeper.connection.timeout.ms", "6000");		
		/**
		 * 配置key的序列化类
		 */
		map.put("key.serializer", StringSerializer.class.getName());
		map.put("value.serializer", StringSerializer.class.getName());
		map.put("bootstrap.servers", "10.1.112.222:9092");        
		return new KafkaProducer(map);
	}

	public static void main(String[] args)
	{
		new TestKafkaProducer("test2").start();// 使用kafka集群中创建好的主题 test
	}

}

4.2 消费者Demo

package com.kafka.test;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import com.test.common.BusinessException;
import com.test.tool.SpeedCounterUtils;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.ConsumerTimeoutException;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;

public class TestKafkaConsumer extends Thread
{
	private static final Logger LOG = Logger.getLogger(TestKafkaConsumer.class);

	/**
	 * 日志信息配置文件
	 */
	public static final String LOG4J_PROPERTIES = "conf/log4j.properties";

	static
	{
		/**
		 * 设置log信息配置文件
		 */
		PropertyConfigurator.configure(LOG4J_PROPERTIES);
	}

	/**
	 * 消费者连接器
	 */
	private final ConsumerConnector consumer;
	
	private KafkaConsumer kafkaConsumer;
	
	
	/**
	 * 总共读取的消息数
	 */
	private static AtomicLong totalMessagesRead = new AtomicLong(0);
	
	
	/**
	 * 总共读取的字节数
	 */
	private static AtomicLong totalBytesRead = new AtomicLong(0);
	
	
	/**
	 * 主题
	 */
	private final String topic;

	/**
	 * 执行服务器
	 */
	private ExecutorService executor;

	public TestKafkaConsumer(String topic)
	{
		Properties properties = new Properties();  
		properties.put("bootstrap.servers", "master1:9092,slave11:9092,slave12:9092,"
				+ "slave13:9092,slave14:9092,slave15:9092,slave16:9092,slave17:9092,slave18:9092");
		properties.put("enable.auto.commit", true);
		properties.put("auto.commit.interval.ms", "100");
		properties.put("session.timeout.ms", "1500000");
		properties.put("key.deserializer", StringDeserializer.class);
		properties.put("value.deserializer", StringDeserializer.class);
		properties.put("auto.offset.reset", "smallest");
		properties.put("group.id", "tdfs196899");
		properties.put("zookeeper.connect", "master1:2181,slave12:2181,slave13:2181,slave14:2181,slave15:2181");
		properties.put("consumer.timeout.ms", "-1");
		properties.put("request.timeout.ms", "40000");
		this.consumer= Consumer.createJavaConsumerConnector(new ConsumerConfig(properties)); 
		this.topic = topic;
	}
	
	public void shutdown()
	{
		if (consumer != null)
			consumer.shutdown();
		if (executor != null)
			executor.shutdown();
	}

	public void run(int numThreads)
	{
		Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
		topicCountMap.put(topic, new Integer(numThreads));
		Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
		List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);
		executor = Executors.newFixedThreadPool(numThreads);
		int threadNumber = 0;
		for (final KafkaStream stream : streams)
		{
			executor.submit(new ConsumerMsgTask(stream));
			threadNumber++;
		}
	}
	
	public static class ConsumerMsgTask implements Runnable
	{
		private KafkaStream kafkaStream;
		
		public ConsumerMsgTask(KafkaStream stream)
		{
			this.kafkaStream = stream;
		}

		public void run()
		{
			ConsumerIterator<byte[], byte[]> iter = kafkaStream.iterator();
			long bytesRead = 0;
			long messagesRead = 0;
			
			try
			{
				while (iter.hasNext())
				{
					MessageAndMetadata<byte[], byte[]> messageAndMetadata = iter.next();
					 String message = new String(messageAndMetadata.message());  
					 System.out.println(message);
					messagesRead += 1;
					bytesRead += messageAndMetadata.message().length;
				}
			}
			catch (ConsumerTimeoutException e)
			{
	            LOG.error("Read Messages fail: "+ e.getMessage(), e);
	            throw new BusinessException("Read Messages fail: "+ e.getMessage(), e);
			}
			catch (Exception e)
			{
				 LOG.error("Read Messages fail: "+ e.getMessage(), e);
				 throw new BusinessException("Read Messages fail: "+ e.getMessage(), e);
			}
			totalMessagesRead.addAndGet(messagesRead);
			totalBytesRead.addAndGet(bytesRead);
		}

	}



	public static void main(String[] arg)
	{

		int threads = 1;
		TestKafkaConsumer testKafkaConsumer = new TestKafkaConsumer("jdctx");
	
		try
		{
			testKafkaConsumer.run(threads);
			Thread.sleep(10000);
		}
		catch (InterruptedException ie)
		{
			
		}
//		testKafkaConsumer.shutdown();
    	
		
		SpeedCounterUtils.startCount("solr插入", totalMessagesRead, 10000, 10000);

	}
}

4.3 常见问题
问题一：序列化化问题
问题描述：key和value序列化无法构造StringEncoder对象。

现象展示：

 
问题原因：参考网上资料需要加以验证才行。
问题解决：使用StringSerializer来替换StringEncoder实现序列化。




问题二：加载.class问题。
问题描述：java无服务加载scala的class文件
现象展示：




问题原因：java会用到scala的一些工具包，但是在引用中没有加载scala编译的包导致此错误。


问题解决：把scala编译用的包引入java中。


问题三：C语言接口发送失败
问题描述：java可以正常发送，但是使用C提供的库包，却发送失败，报hostname命名问题。
现象展示：
 
问题原因：Kafka内部消息IP的屏蔽（还没有完全弄懂）。
问题解决：在8台主机上面分别配置主机名与IP名的映射关系