kafka入门程序

一. 依赖jar包

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.zrj.points</groupId>
  <artifactId>kafka-consumer</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <packaging>war</packaging>
  <name>points-consumer</name>
  <properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<spring.version>4.3.3.RELEASE</spring.version>
  </properties>
  <dependencies>
		<dependency>
		  <groupId>org.slf4j</groupId>
		  <artifactId>slf4j-api</artifactId>
		  <version>1.7.5</version>
		</dependency>
		<dependency>
		  <groupId>org.slf4j</groupId>
		  <artifactId>slf4j-log4j12</artifactId>
		  <version>1.7.5</version>
		</dependency>
		<dependency>
		  <groupId>org.scala-lang</groupId>
		  <artifactId>scala-library</artifactId>
		  <version>2.9.2</version>
		</dependency>
		<dependency>
		  <groupId>com.101tec</groupId>
		  <artifactId>zkclient</artifactId>
		  <version>0.3</version>
		</dependency>
		<dependency>
		  <groupId>org.apache.kafka</groupId>
		  <artifactId>kafka_2.9.2</artifactId>
		  <version>0.8.1.1</version>
		</dependency>
		<dependency>
		  <groupId>org.apache.zookeeper</groupId>
		  <artifactId>zookeeper</artifactId>
		  <version>3.4.5</version>
		</dependency>
		<dependency>
		  <groupId>com.yammer.metrics</groupId>
		  <artifactId>metrics-core</artifactId>
		  <version>2.2.0</version>
		</dependency>
		<dependency>
		  <groupId>commons-logging</groupId>
		  <artifactId>commons-logging</artifactId>
		  <version>1.2</version>
		</dependency>
		<dependency>
		  <groupId>log4j</groupId>
		  <artifactId>log4j</artifactId>
		  <version>1.2.17</version>
		</dependency>
		<dependency>
		  <groupId>commons-lang</groupId>
		  <artifactId>commons-lang</artifactId>
		  <version>2.6</version>
		</dependency>
  </dependencies>
  <build>
		<finalName>kafka-consmer</finalName>
		<plugins>
			<plugin>
				<artifactId>maven-compiler-plugin</artifactId>
				<configuration>
					<source>1.7</source>
					<target>1.7</target>
					<encoding>UTF-8</encoding>
				</configuration>
			</plugin>
			
			<plugin>
				<groupId>org.eclipse.jetty</groupId>
				<artifactId>jetty-maven-plugin</artifactId>
				<version>9.2.11.v20150529</version>
				<configuration>
					<jvmArgs>-Xms456m -Xmx456m -XX:MaxNewSize=456m
						-XX:MaxPermSize=1024m</jvmArgs>
					<scanIntervalSeconds>10</scanIntervalSeconds>
					<webApp>
						<contextPath>/</contextPath>
					</webApp>
					<httpConnector>
						<port>9090</port>
					</httpConnector>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>

二: 生产者

package com.qianbao.kafka;

import java.util.Date;
import java.util.Properties;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

public class KafkaProducer {

	private final Producer<String, String> producer;
	
	private KafkaProducer() {
		Properties props = new Properties();
		
		props.put("metadata.broker.list", Utils.KAFKA_BROKER_LIST);
		
//		props.put("zk.connect", "");
//		props.put("zookeeper.connect", "");

		// 配置value的序列化类
		props.put("serializer.class", "kafka.serializer.StringEncoder");
		// 配置key的序列化类
		props.put("key.serializer.class", "kafka.serializer.StringEncoder");
		props.put("request.required.acks", "-1");
		//指定partition
		props.put("partitioner.class", "com.qianbao.kafka.RoutePartition");

		producer = new Producer<String, String>(new ProducerConfig(props));
	}

	void produce(int messageNo, int count) {
		while (messageNo < count) {
			String key = String.valueOf(messageNo);
			String data = "hello kafka message " + key;
			producer.send(new KeyedMessage<String, String>(Utils.KAFKA_TOPIC_NAME,key, data));
			messageNo++;
		}
	}

	public static void main(String[] args) {
		int messageNo = 1000;
		int count = 1010;

		long startTime = new Date().getTime();
		KafkaProducer kafkaProducer = new KafkaProducer();
		kafkaProducer.produce(messageNo, count);
		long endTime = new Date().getTime();
		System.out.println("spend time:" + (endTime - startTime));
	}
}

三.消费者

package com.qianbao.kafka;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.serializer.StringDecoder;
import kafka.utils.VerifiableProperties;

public class KafkaConsumer {
	private final ConsumerConnector consumer;
	private ExecutorService threadPool;

	private KafkaConsumer() {
		Properties props = new Properties();
		// zookeeper 配置
		props.put("zookeeper.connect", Utils.KAFKA_ZOOKEEPER);

		// group 代表一个消费组
		props.put("group.id", Utils.KAFKA_CONSUMER_GROUP_ID);
		// zk连接超时
		props.put("zookeeper.session.timeout.ms", "40000");
		props.put("zookeeper.sync.time.ms", "200");
		props.put("auto.commit.interval.ms", "1000");
		props.put("auto.offset.reset", "smallest");
		// 序列化类
		props.put("serializer.class", "kafka.serializer.StringEncoder");

		ConsumerConfig config = new ConsumerConfig(props);

		consumer = kafka.consumer.Consumer.createJavaConsumerConnector(config);
	}

	void consume() {
		Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
		topicCountMap.put(Utils.KAFKA_TOPIC_NAME, Utils.KAFKA_TOPIC_PARTITION_NUM);
		// topicCountMap.put(KafkaProducer2.TOPIC, new Integer(1));

		StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
		StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

		Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap,
				keyDecoder, valueDecoder);
		List<KafkaStream<String, String>> partitions = consumerMap.get(Utils.KAFKA_TOPIC_NAME);
		threadPool = Executors.newFixedThreadPool(Utils.KAFKA_TOPIC_PARTITION_NUM);
		int partitionNum = 0;
		for (KafkaStream<String, String> partition : partitions) {
			threadPool.execute(new MessageReader(partition, partitionNum));
			partitionNum++;
		}
	}

	class MessageReader implements Runnable {
		private KafkaStream<String, String> partition;
		private int partitionNum;

		MessageReader(KafkaStream<String, String> partition, int partitionNum) {
			this.partition = partition;
			this.partitionNum = partitionNum;
		}

		public void run() {
			ConsumerIterator<String, String> it = partition.iterator();
			while (true) {
				if (it.hasNext()) {
					String oneLineLog = it.next().message();
					System.out.println(partitionNum + " partition: "+ oneLineLog);
				} else {
					try {
						Thread.sleep(3000);
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
				}
			}
		}

	}

	public static void main(String[] args) {
		new KafkaConsumer().consume();
	}
}

四:自定义分区

package com.qianbao.kafka;

import kafka.producer.Partitioner;
import kafka.utils.VerifiableProperties;

public class RoutePartition implements Partitioner {
	public RoutePartition(VerifiableProperties props) {
	}
	
	public int partition(Object arg0, int arg1) {
		// TODO Auto-generated method stub
		return 1;
	}

}

五:工具类(静态常量类)

package com.qianbao.kafka;

import kafka.producer.Partitioner;
import kafka.utils.VerifiableProperties;

public class RoutePartition implements Partitioner {
	public RoutePartition(VerifiableProperties props) {
	}
	
	public int partition(Object arg0, int arg1) {
		// TODO Auto-generated method stub
		return 1;
	}

}