Kafka学习二

转自：http://blog.csdn.net/desilting/article/details/22873335

安装完kafka后，做了下简单测试

消息生产端：

[java]  view plain copy print ?

1. <span style="font-family:Microsoft YaHei;">import java.util.Properties;  
2. import kafka.javaapi.producer.Producer;  
3. import kafka.producer.KeyedMessage;  
4. import kafka.producer.ProducerConfig;  
5.   
6. /** 
7.  * Created with IntelliJ IDEA. 
8.  * User: pis 
9.  * Date: 14-4-3 
10.  * Time: 上午11:48 
11.  * To change this template use File | Settings | File Templates. 
12.  */  
13. public class ProducerTest {  
14.   
15.         public static void main(String[] args) {  
16.             Properties props = new Properties();  
17.             props.setProperty("metadata.broker.list","192.168.40.134:9092");  
18.             props.setProperty("serializer.class","kafka.serializer.StringEncoder");  
19.             props.put("request.required.acks","1");  
20.             ProducerConfig config = new ProducerConfig(props);  
21.             Producer<String, String> producer = new Producer<String, String>(config);  
22.             KeyedMessage<String, String> data = new KeyedMessage<String, String>("mykafka","test-kafka");  
23.             try {  
24.                 producer.send(data);  
25.             } catch (Exception e) {  
26.                 e.printStackTrace();  
27.             }  
28.             producer.close();  
29.         }  
30. }  
31. </span>  

消息消费端：

[java]  view plain copy print ?

1. <span style="font-family:Microsoft YaHei;">import java.util.HashMap;  
2. import java.util.List;  
3. import java.util.Map;  
4. import java.util.Properties;  
5.   
6. import kafka.consumer.ConsumerConfig;  
7. import kafka.consumer.ConsumerIterator;  
8. import kafka.consumer.KafkaStream;  
9. import kafka.javaapi.consumer.ConsumerConnector;  
10.   
11. /** 
12.  * Created with IntelliJ IDEA. 
13.  * User: pis 
14.  * Date: 14-4-3 
15.  * Time: 下午1:19 
16.  * To change this template use File | Settings | File Templates. 
17.  */  
18. public class ConsumerTest extends Thread{  
19.         private final ConsumerConnector consumer;  
20.         private final String topic;  
21.   
22.         public static void main(String[] args) {  
23.             ConsumerTest consumerThread = new ConsumerTest("mykafka");  
24.             consumerThread.start();  
25.         }  
26.         public ConsumerTest(String topic) {  
27.             consumer =kafka.consumer.Consumer  
28.                     .createJavaConsumerConnector(createConsumerConfig());  
29.             this.topic =topic;  
30.         }  
31.   
32.     private static ConsumerConfig createConsumerConfig() {  
33.         Properties props = new Properties();  
34.         props.put("zookeeper.connect","192.168.40.132:2181");  
35.         props.put("group.id", "0");  
36.         props.put("zookeeper.session.timeout.ms","10000");  
37.         return new ConsumerConfig(props);  
38.     }  
39.   
40.     public void run(){  
41.         Map<String,Integer> topickMap = new HashMap<String, Integer>();  
42.         topickMap.put(topic, 1);  
43.         Map<String, List<KafkaStream<byte[],byte[]>>>  streamMap =consumer.createMessageStreams(topickMap);  
44.         KafkaStream<byte[],byte[]>stream = streamMap.get(topic).get(0);  
45.         ConsumerIterator<byte[],byte[]> it =stream.iterator();  
46.         System.out.println("*********Results********");  
47.         while(it.hasNext()){  
48.             System.err.println("get data:" +new String(it.next().message()));  
49.             try {  
50.                 Thread.sleep(1000);  
51.             } catch (InterruptedException e) {  
52.                 e.printStackTrace();  
53.             }  
54.         }  
55.     }  
56. }  
57. </span>  

将依赖的jar打成包：

在pom.xml文件中添加：

[plain]  view plain copy print ?

1. <span style="font-family:Microsoft YaHei;">   <build>  
2.         <finalName>testKafka</finalName>  
3.         <plugins>  
4.             <plugin>  
5.                 <groupId>org.apache.maven.plugins</groupId>  
6.                 <artifactId>maven-compiler-plugin</artifactId>  
7.                 <version>2.0.2</version>  
8.                 <configuration>  
9.                     <source>1.5</source>  
10.                     <target>1.5</target>  
11.                     <encoding>UTF-8</encoding>  
12.                 </configuration>  
13.             </plugin>  
14.             <plugin>  
15.                 <artifactId>maven-assembly-plugin</artifactId>  
16.                 <version>2.4</version>  
17.                 <configuration>  
18.                     <descriptors>  
19.                         <descriptor>src/main/src.xml</descriptor>  
20.                     </descriptors>  
21.                     <descriptorRefs>  
22.                         <descriptorRef>jar-with-dependencies</descriptorRef>  
23.                     </descriptorRefs>  
24.                 </configuration>  
25.                 <executions>  
26.                     <execution>  
27.                         <id>make-assembly</id> <!-- this is used for inheritance merges -->  
28.                         <phase>package</phase> <!-- bind to the packaging phase -->  
29.                         <goals>  
30.                             <goal>single</goal>  
31.                         </goals>  
32.                     </execution>  
33.                 </executions>  
34.             </plugin>  
35.         </plugins>  
36.     </build></span>  

src.xml文件：

[html]  view plain copy print ?

1. <span style="font-family:Microsoft YaHei;"><?xml version="1.0" encoding="UTF-8"?>  
2. <assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"  
3.           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"  
4.           xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd ">  
5.     <id>jar-with-dependencies</id>  
6.     <formats>  
7.         <format>jar</format>  
8.     </formats>  
9.     <includeBaseDirectory>false</includeBaseDirectory>  
10.     <dependencySets>  
11.         <dependencySet>  
12.             <unpack>false</unpack>  
13.             <scope>runtime</scope>  
14.         </dependencySet>  
15.     </dependencySets>  
16.     <fileSets>  
17.         <fileSet>  
18.             <directory>/lib</directory>  
19.         </fileSet>  
20.     </fileSets>  
21. </assembly></span>  

如果报如下错，可使用2.9.2版本的scala-library及scala-compiler包：

[java]  view plain copy print ?

1. <span style="font-family:Microsoft YaHei;">Exception in thread "main" java.lang.NoClassDefFoundError: scala/Tuple2$mcLL$sp  
2.  at kafka.consumer.ConsumerConfig.<init>(ConsumerConfig.scala:77)  
3.  at kafka.ConsumerTest.createConsumerConfig(ConsumerTest.java:44)  
4.  at kafka.ConsumerTest.<init>(ConsumerTest.java:31)  
5.  at kafka.ConsumerTest.main(ConsumerTest.java:27)  
6.  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  
7.  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)  
8.  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)  
9.  at java.lang.reflect.Method.invoke(Method.java:597)  
10.  at com.intellij.rt.execution.application.AppMain.main(AppMain.java:120)  
11. Caused by: java.lang.ClassNotFoundException: scala.Tuple2$mcLL$sp  
12.  at java.net.URLClassLoader$1.run(URLClassLoader.java:202)  
13.  at java.security.AccessController.doPrivileged(Native Method)  
14.  at java.net.URLClassLoader.findClass(URLClassLoader.java:190)  
15.  at java.lang.ClassLoader.loadClass(ClassLoader.java:306)  
16.  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)  
17.  at java.lang.ClassLoader.loadClass(ClassLoader.java:247)  
18.  ... 9 more</span>  

在项目目录下执行mvn package命令，得到依赖包testKafka-jar-with-dependencies.jar

编译：

javac -classpath testKafka-jar-with-dependencies.jar ProducerTest.java

javac -classpath testKafka-jar-with-dependencies.jar ConsumerTest.java

执行：

java -classpath .:testKafka-jar-with-dependencies.jar ProducerTest

java -classpath .:testKafka-jar-with-dependencies.jar CoumerTest

结果：

*********Results********

get message: test-kafka

get message: test-kafka

get message: test-kafka