使用Java代码实现实时消费kafka的消息

首先maven构建开发项目，配置pom.xml文件

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">   <modelVersion>4.0.0</modelVersion>   <groupId>om.cctsoft</groupId>   <artifactId>kafkaTest</artifactId>   <version>0.0.1-SNAPSHOT</version>   <packaging>jar</packaging>   <repositories>     <repository>       <id>java.net</id>       <url>hhttp://repo.springsource.org/libs-milestone</url>     </repository>     <repository>       <id>repo2</id>       <url>http://repo2.maven.org/maven2/</url>     </repository>     <repository>         <id>clojars.org</id>         <url>http://clojars.org/repo</url>     </repository>  </repositories>     <properties>     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>   </properties> <dependencies>         <dependency>           <groupId>junit</groupId>           <artifactId>junit</artifactId>           <version>3.8.1</version>           <scope>test</scope>         </dependency>         <dependency>             <groupId>org.apache.hadoop</groupId>             <artifactId>hadoop-common</artifactId>             <version>2.2.0</version>             <exclusions>                 <exclusion>                   <groupId>org.slf4j</groupId>                   <artifactId>slf4j-log4j12</artifactId>                 </exclusion>                         <exclusion>                   <groupId>log4j</groupId>                   <artifactId>log4j</artifactId>                 </exclusion>             </exclusions>         </dependency>         <dependency>             <groupId>org.apache.hadoop</groupId>             <artifactId>hadoop-hdfs</artifactId>             <version>2.2.0</version>         </dependency>         <dependency>             <groupId>org.apache.hadoop</groupId>             <artifactId>hadoop-client</artifactId>             <version>2.2.0</version>             <exclusions>                 <exclusion>                   <groupId>org.slf4j</groupId>                   <artifactId>slf4j-log4j12</artifactId>                 </exclusion>                         <exclusion>                   <groupId>log4j</groupId>                   <artifactId>log4j</artifactId>                 </exclusion>             </exclusions>         </dependency>         <dependency>             <groupId>org.apache.hbase</groupId>             <artifactId>hbase-client</artifactId>             <version>1.0.3</version>         </dependency>         <dependency>             <groupId>org.apache.hbase</groupId>             <artifactId>hbase-server</artifactId>             <version>1.0.3</version>         </dependency>         <dependency>             <groupId>org.apache.hadoop</groupId>             <artifactId>hadoop-hdfs</artifactId>             <version>2.2.0</version>         </dependency>         <dependency>             <groupId>jdk.tools</groupId>             <artifactId>jdk.tools</artifactId>             <version>1.7</version>             <scope>system</scope>             <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>         </dependency>         <dependency>             <groupId>org.apache.httpcomponents</groupId>             <artifactId>httpclient</artifactId>             <version>4.3.6</version>         </dependency>         <dependency>             <groupId>org.slf4j</groupId>             <artifactId>slf4j-log4j12</artifactId>             <version>1.7.21</version>         </dependency>         <dependency>             <groupId>net.java.dev.jets3t</groupId>             <artifactId>jets3t</artifactId>             <version>0.9.4</version>         </dependency>         <!-- Storm -->         <dependency>             <groupId>org.apache.storm</groupId>             <artifactId>storm-core</artifactId>             <version>0.9.6</version>             <scope>provided</scope>             <exclusions>                 <exclusion>                     <groupId>org.slf4j</groupId>                     <artifactId>log4j-over-slf4j</artifactId>                 </exclusion>             </exclusions>         </dependency>         <dependency>             <groupId>org.apache.storm</groupId>             <artifactId>storm-kafka</artifactId>             <version>0.9.6</version>               <scope>provided</scope>         </dependency>                 <dependency>             <groupId>org.apache.kafka</groupId>             <artifactId>kafka_2.11</artifactId>             <version>0.10.1.1</version>             <exclusions>                 <exclusion>                     <groupId>org.apache.zookeeper</groupId>                     <artifactId>zookeeper</artifactId>                 </exclusion>                 <exclusion>                   <groupId>org.slf4j</groupId>                   <artifactId>slf4j-log4j12</artifactId>                 </exclusion>                 <exclusion>                     <groupId>log4j</groupId>                     <artifactId>log4j</artifactId>                 </exclusion>             </exclusions>         </dependency>              </dependencies> <build>     <plugins>       <plugin>           <artifactId>maven-assembly-plugin</artifactId>           <configuration>               <appendAssemblyId>false</appendAssemblyId>               <descriptorRefs>                   <descriptorRef>jar-with-dependencies</descriptorRef>               </descriptorRefs>               <archive>                   <manifest>                       <mainClass>om.cctsoft.kafka.customer.storm.MyKafkaBolt</mainClass>                   </manifest>               </archive>           </configuration>           <executions>               <execution>                   <id>make-assembly</id>                   <phase>package</phase>                   <goals>                       <goal>assembly</goal>                   </goals>               </execution>           </executions>         </plugin>     </plugins>   </build> </project>

kafka服务信息配置kafka.properties

zookeeper.connect=vm12:2181,vm13:2181,vm14:2181 group.id=liuygTest   zookeeper.session.timeout.ms=4000   zookeeper.sync.time.ms=200   auto.commit.interval.ms=1000   auto.offset.reset=smallest   serializer.class=kafka.serializer.StringEncoder

java代码实现

package om.cctsoft.kafka.customer.java; import java.io.File; import java.io.FileInputStream; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import kafka.consumer.ConsumerConfig; import kafka.consumer.ConsumerIterator; import kafka.consumer.KafkaStream; import kafka.javaapi.consumer.ConsumerConnector; import kafka.serializer.StringDecoder; import kafka.utils.VerifiableProperties; public class KafkaConsume {     private final static String TOPIC = "cctsoft1102";     private static Properties properties;     static {         properties = new Properties();         String path = "D:\\src\\eclipse-workspace20171025\\kafkaTest\\src\\main\\java\\kafka.properties";         System.out.println(path);         try {             FileInputStream fis = new FileInputStream(new File(path));             properties.load(fis);         } catch (Exception e) {             e.printStackTrace();         }     }     /**      * 获取消息      *      * @throws Exception      */     public void getMsg() throws Exception {         ConsumerConfig config = new ConsumerConfig(properties);         ConsumerConnector consumer = kafka.consumer.Consumer                 .createJavaConsumerConnector(config);         Map<String, Integer> topicCountMap = new HashMap<String, Integer>();         topicCountMap.put(TOPIC, new Integer(1));         StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());         StringDecoder valueDecoder = new StringDecoder(                 new VerifiableProperties());         Map<String, List<KafkaStream<String, String>>> consumerMap = consumer                 .createMessageStreams(topicCountMap, keyDecoder, valueDecoder);         KafkaStream<String, String> stream = consumerMap.get(TOPIC).get(0);         ConsumerIterator<String, String> it = stream.iterator();         while (it.hasNext()) {             String json = it.next().message();                          System.out.println(json);         }     }     public static void main(String[] args) {         // TODO Auto-generated method stub         try {             new KafkaConsume().getMsg();         } catch (Exception e) {             // TODO Auto-generated catch block             e.printStackTrace();         }     } }