kafka在ssm中的应用

kafka在ssm中的应用

kafka也是在dubbo一样想注册中心进行注册进行监控如使用zookeeper中注册产生节点偏移量进行记录生产者想注册中心发送带有一个标识消费者在要消费时到注册中心中拿相应需要的数据。

在使用kafka是要在Linux中启用zookeeper和kafka的生产者和消费者

在pom文件中引入kafka的jar包配置

<!-- https://mvnrepository.com/artifact/org.springframework.kafka/spring-kafka -->
		<dependency>
			<groupId>org.springframework.kafka</groupId>
			<artifactId>spring-kafka</artifactId>
			<version>2.2.0.RELEASE</version>
		</dependency>


		<dependency>
			<groupId>org.apache.kafka</groupId>
			<artifactId>kafka_2.10</artifactId>
			<version>0.8.2.1</version>
			<exclusions>
				<exclusion>
					<artifactId>jmxri</artifactId>
					<groupId>com.sun.jmx</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jms</artifactId>
					<groupId>javax.jms</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jmxtools</artifactId>
					<groupId>com.sun.jdmk</groupId>
				</exclusion>
			</exclusions>
		</dependency>

kafka生产者的xml配置文件

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
	xmlns:context="http://www.springframework.org/schema/context"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.springframework.org/schema/beans 
	    http://www.springframework.org/schema/beans/spring-beans.xsd
	    http://www.springframework.org/schema/context
	    http://www.springframework.org/schema/context/spring-context.xsd">

	<!--参数配置 -->
	<bean id="producerProperties" class="java.util.HashMap">
		<constructor-arg>
			<map>
				<!-- kafka服务地址，可能是集群 value="localhost:9092,localhost:9093,localhost:9094"-->
				<entry key="bootstrap.servers" value="192.168.226.168:9092" />
				<!-- 组id标志 -->
				<entry key="group.id" value="test-consumer-group" />
				<!-- 有可能导致broker接收到重复的消息-->
				<entry key="retries" value="0" />
				<!-- 每次批量发送消息的数量 -->
				<entry key="batch.size" value="1638" />
				<!-- 默认0ms，在异步IO线程被触发后（任何一个topic，partition满都可以触发） -->
				<entry key="linger.ms" value="1" />
				<!--producer可以用来缓存数据的内存大小。如果数据产生速度大于向broker发送的速度，producer会阻塞或者抛出异常 -->
				<entry key="buffer.memory" value="33554432 " />
				
				<entry key="key.serializer"
					value="org.apache.kafka.common.serialization.StringSerializer" />
					
				<entry key="value.serializer"
					value="org.apache.kafka.common.serialization.StringSerializer" />
			</map>
		</constructor-arg>
	</bean>

	<!-- 创建kafkatemplate需要使用的producerfactory bean -->
	<bean id="producerFactory"
		class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
		<constructor-arg>
			<ref bean="producerProperties" />
		</constructor-arg>
	</bean>

	<!-- 创建kafkatemplate bean，使用的时候，只需要注入这个bean，即可使用template的send消息方法 -->
	<bean id="KafkaTemplate"
		class="org.springframework.kafka.core.KafkaTemplate">
		<constructor-arg ref="producerFactory" />
		<!--设置对应topic -->
		<property name="defaultTopic" value="one1" />
	</bean>
</beans>		

kafka消费者的xml配置文件

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
	xmlns:context="http://www.springframework.org/schema/context"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.springframework.org/schema/beans 
	    http://www.springframework.org/schema/beans/spring-beans.xsd
	    http://www.springframework.org/schema/context
	    http://www.springframework.org/schema/context/spring-context.xsd">

	<bean id="consumerProperties" class="java.util.HashMap">
		<constructor-arg>
			<map>
				<!--Kafka服务地址 -->
				<entry key="bootstrap.servers" value="192.168.226.168:9092" />
				<!--Consumer的组ID，相同group.id的consumer属于同一个组。 -->
				<entry key="group.id" value="test-consumer-group" />
				<!--如果此值设置为true，consumer会周期性的把当前消费的offset值保存到zookeeper。当consumer失败重启之后将会使用此值作为新开始消费的值。 -->
				<entry key="enable.auto.commit" value="true" />
				<!--网络请求的socket超时时间。实际超时时间由max.fetch.wait + socket.timeout.ms 确定 -->
				<entry key="session.timeout.ms" value="15000 " />
				
				<entry key="key.deserializer"
					value="org.apache.kafka.common.serialization.StringDeserializer" />
					
				<entry key="value.deserializer"
					value="org.apache.kafka.common.serialization.StringDeserializer" />
			</map>
		</constructor-arg>
	</bean>


	<!-- 创建consumerFactory bean -->
	<bean id="consumerFactory"
		class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
		<constructor-arg>
			<ref bean="consumerProperties" />
		</constructor-arg>
	</bean>
	
	
	
	<bean id="containerProperties"
		class="org.springframework.kafka.listener.ContainerProperties">
		<constructor-arg value="one1" />
		<property name="messageListener"
			ref="messageListernerConsumerService" />
	</bean>
	
	
	<bean id="messageListenerContainer"
		class="org.springframework.kafka.listener.KafkaMessageListenerContainer"
		init-method="doStart">
		<constructor-arg ref="consumerFactory" />
		<constructor-arg ref="containerProperties" />
	</bean>
	

	
	<!--指定具体监听类的bean -->
	<bean id="messageListernerConsumerService"
		class="com.liujunyang.kafka.KafkaConsumerListener" />

</beans>		

kafka的生产者

@Autowired
	private KafkaTemplate<String, String> kafkaTemplate;
	
private ObjectMapper objectMapper = new ObjectMapper();
	
	private static final Logger LOGGER = Logger.getLogger(ArticleServiceImpl.class);


@Override
	public ResultTO articlePush(Article article,String[] inpus) {
		try {	
			//转成json类型
			String json = objectMapper.writeValueAsString(article);
			//放入到kafka的send中进行发送
			kafkaTemplate.sendDefault("article_" + article.getUserId(), json);
		} catch (Exception e) {
			e.printStackTrace();
			return ResultTO.newFailResultTO("发布失败", null);
		}
		return ResultTO.newSuccessResultTO("发布成功", null);
	}

kafka的监控中心

public class KafkaConsumerListener implements MessageListener<String, String> {

	
	private static final Logger LOGGER = Logger.getLogger(KafkaConsumerListener.class);
	
	@Autowired
	private UserService userService;
	@Autowired
	private LeaveService leaveService;
	@Autowired
	private ArticleMapper articleMapper;
	@Autowired
	private ArticleService articleService;
	
	private ObjectMapper objectMapper = new ObjectMapper();
	
	@Override
	public void onMessage(ConsumerRecord<String, String> data) {
		LOGGER.info("开始消息监听------------------------------{}");
		LOGGER.info("接收到消息"+data.value());
		LOGGER.info(data.topic()+"@@"+data.offset()+"@@"+data.key());
		LOGGER.info("消息监听结束------------------------------{}");
		
		try {
			String key = data.key();
			LOGGER.info("----------------------------------{}@@@@@@"+key);
			if (key != null) {
				if (key.contains("article_")) {
					String json = data.value();
					Article article = objectMapper.readValue(json, Article.class);
					articleMapper.articlePush(article);
					LOGGER.info("kafka文章添加---------------------------------------------------------------{}");
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		LOGGER.info("kafka消费者处理完成--------------------------{}");
	}

}