1.项目构建
主机（kafka zookeeper 三台同时安装启动）
master: 192.168.28.129
slave0: 192.168.28.130
slave1: 192.168.28.131
kafka zookeeper 启动进程
master:
slave0:
slave1:
2.pom.xml 依赖
导入依赖时需要注意版本问题
<dependencies>
<!-- 单元测试 -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<!-- 导入servlet支持包 -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>4.0.0-b01</version>
</dependency>
<!-- 导入日志支持包 -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<!-- 日志系统支持(slf4j-log4j12:链接slf4j-api和log4j中间的适配器。它实现了slf4j-apiz中StaticLoggerBinder接口，
从而使得在编译时绑定的是slf4j-log4j12的getSingleton()方法) -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.21</version>
</dependency>
<!-- spring4相关jar包导入 -->
<!-- 提供在基础IOC(控制反转)功能上的扩展服务，此外还提供许多企业级服务的支持，有邮件服务、
任务调度、JNDI定位，EJB集成、远程访问、缓存以及多种视图层框架的支持。 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- spring核心工具包,其它包依赖此包 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- spring对jdbc的简单封闭包 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- spring用于支持web开发时所需的Spring框架的核心类，包括自动载入WebApplicationContext特性的类、
Struts与JSF集成类、文件上传的支持类、Filter类和大量工具辅助类。 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- 用于对spring AOP(面向切面)的支持jar包 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aop</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- 扩展包:spring context 的扩展支持,用于MVC方面-->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- spring bean管理包(包括配置文件的访问和创建bean管理bean)是Spring IOC的基础实现-->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- spring提供对事物的支持,事务的相关处理以及实现类就在这个jar包中 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-tx</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- 提供对AspectJ的支持，以便可以方便的将面向方面的功能集成进IDE中，比如Eclipse AJDT。
外部依赖。 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aspects</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!-- springMvc4对json的支持 -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.5.0</version>
</dependency>
<!-- 这个jar 文件包含Spring MVC 框架相关的所有类。
包括框架的Servlets，Web MVC框架，控制器和视图支持。当然，如果你的应用使用了独立的MVC 框架，则无需这个JAR 文件里的任何类。
外部依赖spring-web, (spring-support，Tiles，iText，POI)。 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>4.3.2.RELEASE</version>
</dependency>
<!--Spring数据库访问中另外一大模块就是ORM，ORM即对象/关系映射。
Spring支持大多数ORM框架，比如Hibernate，JPA，JDO，TopLink和iBatis
（Spring2支持iBatis2，现MyBatis3的Spring支持由MyBatis社区开发，并非Spring）。 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
<version>4.0.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-config</artifactId>
<version>4.0.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-web</artifactId>
<version>4.0.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>
<version>4.0.2.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>4.1.6.RELEASE</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>1.5.2</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
<version>1.8.7</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-servlet-api</artifactId>
<version>8.0.26</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-frontend-jaxrs</artifactId>
<version>3.1.2</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
<version>1.9.2</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>1.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.0</version>
</dependency>
<!-- 导入servlet支持包 -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>3.0.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
<version>2.1</version>
<scope>provided</scope>
</dependency>
<!-- 导入jstl支持包 -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>jstl</artifactId>
<version>1.2</version>
</dependency>
<!-- kafka客户端支持包 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.11.0.1</version>
</dependency>
<!-- spring支持kafka -->
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>2.0.4.RELEASE</version>
</dependency>
</dependencies>3. config.properties配置文件
#============================= kafka ============================#
#主题
#页面浏览 topic
kafka.topic_page=pageTopic
#点击 topic
kafka.topic_click=clickTopic
#kafka 集群地址
kafka.servers=192.168.28.129:9092,192.168.28.130:9092,192.168.28.131:9092
#指定组名
kafka.group_id_report=report-data
kafka.report_data_container="reportDataContainer1"4.application-kafka.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:jee="http://www.springframework.org/schema/jee"
xmlns:tx="http://www.springframework.org/schema/tx"
xmlns:task="http://www.springframework.org/schema/task"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xmlns:rabbit="http://www.springframework.org/schema/rabbit"
xsi:schemaLocation="
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-4.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.0.xsd
http://www.springframework.org/schema/jee http://www.springframework.org/schema/jee/spring-jee-4.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-4.0.xsd
http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-4.1.xsd
http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc.xsd
http://www.springframework.org/schema/rabbit http://www.springframework.org/schema/rabbit/spring-rabbit-2.1.xsd
">
<!-- 加载配置文件 -->
<context:property-placeholder ignore-unresolvable="false" location="classpath:properties/config.properties"/>
<!-- 生产者 -->
<bean id="producerFactory" class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
<constructor-arg>
<map>
<entry key="bootstrap.servers" value="${kafka.servers}"></entry>
<entry key="key.serializer" value="org.apache.kafka.common.serialization.StringSerializer"></entry>
<entry key="value.serializer" value="org.apache.kafka.common.serialization.StringSerializer"></entry>
<entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"></entry>
<entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"></entry>
</map>
</constructor-arg>
</bean>
<!-- 用于发送消息的模板 kafkaTemplate-->
<bean id="kafkaTemplate" class="org.springframework.kafka.core.KafkaTemplate">
<constructor-arg ref="producerFactory"></constructor-arg>
<constructor-arg name="autoFlush" value="true"></constructor-arg>
</bean>
<!-- 创建消费者 -->
<bean id="consumerFactory" class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
<constructor-arg>
<map>
<entry key="bootstrap.servers" value="${kafka.servers}"></entry>
<entry key="group.id" value="${kafka.report_data_container}"></entry>
<entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"></entry>
<entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"></entry>
</map>
</constructor-arg>
</bean>
<!-- 消息消费者监听器 用于在接收到消息时执行相关的业务代码 (自行定义的类 用户于处理相应的业务操作)-->
<bean id="consumerListener" class="com.java.kafka.listener.KafkaConsumerListener"/>
<bean id="reportDataListener" class="com.java.kafka.listener.ReportDataListener"></bean>
<!-- 用于将监听器和个某个主题绑定在一起 -->
<bean id="containerProperties_example" class="org.springframework.kafka.listener.config.ContainerProperties">
<!-- <constructor-arg value="test-topic"></constructor-arg>
<property name="messageListener" ref="consumerListener"></property> -->
<property name="messageListener" ref="reportDataListener"/>
<constructor-arg>
<list>
<value>${kafka.topic_page}</value>
<!-- <value>${kafka.topic_click}</value> -->
</list>
</constructor-arg>
</bean>
<!-- 用于将消息监听器与消息消费者工厂绑定在一起 -->
<bean id="messageListtenerContainer_example" class="org.springframework.kafka.listener.KafkaMessageListenerContainer">
<constructor-arg ref="containerProperties_example"></constructor-arg>
<constructor-arg ref="consumerFactory"></constructor-arg>
</bean>
<!-- springMvc4 用于将对象转换为 JSON -->
<bean id="stringConverter"
class="org.springframework.http.converter.StringHttpMessageConverter">
<property name="supportedMediaTypes">
<list>
<value>text/plain;charset=UTF-8</value>
</list>
</property>
</bean>
<bean id="jsonConverter"
class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter"></bean>
<bean
class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
<property name="messageConverters">
<list>
<ref bean="stringConverter" />
<ref bean="jsonConverter" />
</list>
</property>
</bean>
</beans> 5.spring-mvc.xml
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:aop="http://www.springframework.org/schema/aop"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:jee="http://www.springframework.org/schema/jee"
xmlns:tx="http://www.springframework.org/schema/tx"
xsi:schemaLocation="
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-4.0.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.0.xsd
http://www.springframework.org/schema/jee http://www.springframework.org/schema/jee/spring-jee-4.0.xsd
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-4.0.xsd">
<!-- 开启注解扫描功能 springMvc-->
<context:component-scan base-package="com.java.kafka.controller" />
<!-- 视图解析器 (对应DispatcherServlet跳转(逻辑视图名))-->
<bean id="viewResolver"
class="org.springframework.web.servlet.view.InternalResourceViewResolver">
<property name="prefix" value="/WEB-INF" />
<property name="suffix" value=".jsp"></property>
</bean>
</beans>6 消息发送者
package com.java.kafka.util;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.kafka.core.KafkaTemplate;
/**
*
* @ClassName: SpringProducer
* @Description: TODO(消息发送者)
* @author lenovo
* @date 2019年1月12日
*
*/
public class SpringProducer {
public static void main(String[] args) {
// TODO Auto-generated method stub
ApplicationContext context = new ClassPathXmlApplicationContext("application-kafka.xml");
KafkaTemplate<String, String> kafkaTemplate = (KafkaTemplate<String, String>)context.getBean("kafkaTemplate");
kafkaTemplate.send("test-topic","消息中间件实践");
kafkaTemplate.send("test-topic","docker技术");
kafkaTemplate.send("test-topic","springcloud实践");
}
}
7.消费者监听类
package com.java.kafka.util;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.listener.MessageListener;
import org.springframework.kafka.support.Acknowledgment;
/**
*
* @ClassName: KafkaConsumerListener
* @Description: TODO(消息消费者)
* @author lenovo
* @date 2019年1月12日
*
*/
public class KafkaConsumerListener implements MessageListener<String, String> {
public void onMessage(ConsumerRecord<String, String> record) {
// TODO Auto-generated method stub
System.out.println("监听开始.......");
System.out.println("partition = " + record.partition()
+ " ,offset = " + record.offset()
+ " ,key = " + record.key() + " ,value = " + record.value());
}
public void onMessage(ConsumerRecord<String, String> data,
Acknowledgment acknowledgment) {
// TODO Auto-generated method stub
}
public void onMessage(ConsumerRecord<String, String> data,
Consumer<?, ?> consumer) {
// TODO Auto-generated method stub
}
public void onMessage(ConsumerRecord<String, String> data,
Acknowledgment acknowledgment, Consumer<?, ?> consumer) {
// TODO Auto-generated method stub
}
}package com.java.kafka.util;
public class ContentTool {
public static final String TOPIC_CLICK = "clickTopic";
public static final String TOPIC_PAGE = "pagePage";
public static final String COMMODITY_Topic = "commodityTopic";
public static final String COMMOD_KEY_01 = "commod_key_01";
public static final String STOCK = "10";
}8.web.xml
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://java.sun.com/xml/ns/javaee"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" id="WebApp_ID" version="3.0">
<display-name>KafkaS</display-name>
<welcome-file-list>
<welcome-file>index.jsp</welcome-file>
</welcome-file-list>
<servlet-mapping>
<servlet-name>default</servlet-name>
<url-pattern>*.js</url-pattern>
</servlet-mapping>
<servlet-mapping>
<servlet-name>default</servlet-name>
<url-pattern>*.css</url-pattern>
</servlet-mapping>
<!-- Spring监听器 -->
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<!-- 防止spring内存溢出监听器 -->
<listener>
<listener-class>org.springframework.web.util.IntrospectorCleanupListener</listener-class>
</listener>
<!-- 监听servletContext，启动contextConfigLocation中的spring配置信息 -->
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<!-- spring 配置 加载配置文件 -->
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>classpath:application-kafka.xml</param-value>
</context-param>
<!-- 编码过滤器 -->
<filter>
<filter-name>encodingFilter</filter-name>
<filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>
<async-supported>true</async-supported>
<init-param>
<param-name>encoding</param-name>
<param-value>UTF-8</param-value>
</init-param>
</filter>
<filter-mapping>
<filter-name>encodingFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<!-- 添加对springmvc的支持 -->
<servlet>
<servlet-name>springMVC</servlet-name>
<servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
<init-param>
<param-name>contextConfigLocation</param-name>
<param-value>classpath:spring-mvc.xml</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
<async-supported>true</async-supported>
</servlet>
<servlet-mapping>
<servlet-name>springMVC</servlet-name>
<url-pattern>*.do</url-pattern>
</servlet-mapping>
<!-- 启用Web监控统计功能需要在Web应用的web.xml中加入这个Servlet声明 -->
<servlet>
<servlet-name>DruidStatView</servlet-name>
<servlet-class>com.alibaba.druid.support.http.StatViewServlet</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>DruidStatView</servlet-name>
<url-pattern>/druid/*</url-pattern>
</servlet-mapping>
</web-app>
8.启动项目，运行消息者类 SpringProducer类 相应 消息者监听类KafkaConsumerListener 消费结果：
9.用户行为数据采集
请求接口地址类（ReportPcController）:
url : http://localhost:8080/KafkaS/report/pc/pageView.do
/**
*
* @Title: pageView
* @Description: TODO(后台请求数据)
* @param @param tempData
* @param @return 参数
* @return Map<String,Object> 返回类型
* @throws
*/
@RequestMapping(value = "pageView.do",method = RequestMethod.POST)
@ResponseBody
public Map<String,Object> pageView(@RequestBody ReportData reportData){
System.out.println("开始上报数据********************");
System.out.println("上报的数据内容【"+reportData.toString()+"】");
kafkaTemplate.send(ContentTool.TOPIC_PAGE,reportData.toString());
Map<String,Object> result = new HashMap<String, Object>();
result.put("success", true);
result.put("data", "页面数据上报成功");
return result;
}10.测试
启动消费者
服务器：192.168.28.131
./kafka-console-consumer.sh --bootstrap-server master:9092 --from-beginning --topic pagePage
消息发送地址请求
消息结果
11.log4j 依赖
<!-- kafka 集成 log4j 日志包 -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-log4j-appender</artifactId>
<version>1.1.0</version>
</dependency>12.日志信息采集 log4j.properties
#log4j.rootLogger=DEBUG,INFO, Console
log4j.rootLogger=INFO,Console,KAFKA
#Console 配置日志输出目的地 控制台输出
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.target=System.out
log4j.appender.Console.encoding=UTF-8
#配置日志信息的格式
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n
#配置日志信息输出目的地 为 kafka (将kafka作为日志输出目的地)
log4j.appender.KAFKA=org.apache.kafka.log4jappender.KafkaLog4jAppender
#将日志数据发送到Kafka时的具体主题名
log4j.appender.KAFKA.topic=log4j-topic
#配置kafka服务器的地址,如果是群集则用逗号分隔
log4j.appender.KAFKA.brokerList=192.168.28.129:9092,192.168.28.130:9092,192.168.28.131:9092
#日志信息是否压缩
log4j.appender.KAFKA.compressionType=none
#表示是否同步发送
log4j.appender.KAFKA.syncSend=true
#将日志输出到Kafka时格式化
log4j.appender.KAFKA.layout=org.apache.log4j.PatternLayout
log4j.appender.kafka.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
#log4j.logger.java.sql.ResultSet=INFO
#log4j.logger.org.apache=INFO
#log4j.logger.java.sql.Connection=DEBUG
#log4j.logger.java.sql.Statement=DEBUG
#log4j.logger.java.sql.PreparedStatement=DEBUG
#log4j.logger.advertisement =TRACE
13. java 测试类
package com.java.kafka.test;
import org.apache.log4j.Logger;
public class KafkaLog4jTest1 {
private static Logger logger = Logger.getLogger(KafkaLog4jTest1.class);
public static void main(String[] args) throws InterruptedException {
for (int i = 0; i < 5; i++) {
logger.info("这是从 KafkaLog4jTest1 产生的消息 【"+i+"】...");
Thread.sleep(1000);
}
}
}
########################################################################################
package com.java.kafka.test;
import org.apache.log4j.Logger;
public class KafkaLog4jTest2 {
private static Logger logger = Logger.getLogger(KafkaLog4jTest2.class);
public static void main(String[] args) throws InterruptedException {
// TODO Auto-generated method stub
for (int i = 0; i < 5; i++) {
logger.info("这是从 KafkaLog4jTest2 产生的消息 【"+i+"】...");
Thread.sleep(1000);
}
}
}
14.测试
启动消费者
服务器：192.168.28.131
./kafka-console-consumer.sh --bootstrap-server master:9092 --from-beginning --topic log4j-topic
分别运行 KafkaLog4jTest1 ，KafkaLog4jTest2 两个测试类
消息消费如下:
1878
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
