添加rabbitMq依赖
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
配置rabbitMq 服务器
服务端和消费端都要配置
#配置rabbitMq 服务器
spring.rabbitmq.host= 10.10.240.xxx
spring.rabbitmq.port= 5672
spring.rabbitmq.username= chijiaqi
spring.rabbitmq.password= 123456
服务端配置类
package cn.edu.nfu.jw.conf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.*;
import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
/*
* @Author : chijiaqi
* @CreateTime : 2020/4/14
* @Description :
*/
/**
Broker:它提供一种传输服务,它的角色就是维护一条从生产者到消费者的路线,保证数据能按照指定的方式进行传输,
Exchange:消息交换机,它指定消息按什么规则,路由到哪个队列。
Queue:消息的载体,每个消息都会被投到一个或多个队列。
Binding:绑定,它的作用就是把exchange和queue按照路由规则绑定起来.
Routing Key:路由关键字,exchange根据这个关键字进行消息投递。
vhost:虚拟主机,一个broker里可以有多个vhost,用作不同用户的权限分离。
Producer:消息生产者,就是投递消息的程序.
Consumer:消息消费者,就是接受消息的程序.
Channel:消息通道,在客户端的每个连接里,可建立多个channel.
*/
@Configuration
public class RabbitConfig {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
@Value("${spring.rabbitmq.host}")
private String host;
@Value("${spring.rabbitmq.port}")
private int port;
@Value("${spring.rabbitmq.username}")
private String username;
@Value("${spring.rabbitmq.password}")
private String password;
public static final String JW_EXCHANGE_DIRECT = "my-mq-exchange_A";
public static final String JW_QUEUE_DIRECT = "JwDirectQueue";
public static final String JW_ROUTINGKEY_DIRECT = "spring-boot-routingKey_A";
@Bean
public ConnectionFactory connectionFactory() {
CachingConnectionFactory connectionFactory = new CachingConnectionFactory(host,port);
connectionFactory.setUsername(username);
connectionFactory.setPassword(password);
connectionFactory.setVirtualHost("/");
connectionFactory.setPublisherConfirms(true);
return connectionFactory;
}
@Bean
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//必须是prototype类型
public RabbitTemplate rabbitTemplate() {
RabbitTemplate template = new RabbitTemplate(connectionFactory());
return template;
}
/**
* 针对消费者配置
* 1. 设置交换机类型
* 2. 将队列绑定到交换机
FanoutExchange: 将消息分发到所有的绑定队列,无routingkey的概念
HeadersExchange :通过添加属性key-value匹配
DirectExchange:按照routingkey分发到指定队列
TopicExchange:多关键字匹配
*/
@Bean
public DirectExchange defaultExchange() {
return new DirectExchange(JW_EXCHANGE_DIRECT);
}
/**
* 获取队列A
* @return
*/
@Bean
public Queue queueA() {
return new Queue(JW_QUEUE_DIRECT, true); //队列持久
}
//一个交换机可以绑定多个消息队列,也就是消息通过一个交换机,可以分发到不同的队列当中去。
@Bean
public Binding binding() {
return BindingBuilder.bind(queueA()).to(defaultExchange()).with(JW_ROUTINGKEY_DIRECT);
}
}
服务端发送消息
@Autowired
RabbitTemplate rabbitTemplate; //使用RabbitTemplate,这提供了接收/发送等等方法
rabbitTemplate.convertAndSend(RabbitConfig.JW_EXCHANGE_DIRECT, RabbitConfig.JW_ROUTINGKEY_DIRECT, JSON.toJSONString(jwlog));//jwlog为消息实体类
消费端配置线程池
package cn.edu.nfu.jw.log.conf;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.ThreadPoolExecutor;
/**
* @Auther: cjq
* @Date: 2020/4/15 11:33
* @Description:
* @version:V1.0
* 用到线程池的时候还需要创建Executors,spring中有十分优秀的支持,
* 就是注解@EnableAsync就可以使用多线程,@Async加在线程任务的方法上(需要异步执行的任务),
* 定义一个线程任务,通过spring提供的ThreadPoolTaskExecutor就可以使用线程池
*
*/
@Configuration
@EnableAsync
public class ThreadPoolConf {
@Bean
public ThreadPoolTaskExecutor logThreadPoolTaskExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 设置核心线程数,默认为1
executor.setCorePoolSize(10);
// 设置最大线程数,默认为Integer.MAX_VALUE
executor.setMaxPoolSize(50);
// 设置队列容量,一般需要设置值大于等于notifyScheduledMainExecutor.maxNum;默认为Integer.MAX_VALUE
// executor.setQueueCapacity(2);
// 设置线程池维护线程所允许的空闲时间,默认为60s
executor.setKeepAliveSeconds(300);
// 设置默认线程名称
executor.setThreadNamePrefix("jw-thread-");
// 设置拒绝策略rejection-policy:当pool已经达到max size的时候,如何处理新任务 CALLER_RUNS:不在新线程中执行任务,而是有调用者所在的线程来执行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 等待所有任务结束后再关闭线程池
executor.setWaitForTasksToCompleteOnShutdown(true);
return executor;
}
}
线程池业务实现类
日志存储服务接口类
package cn.edu.nfu.jw.log.service;
import cn.edu.nfu.jw.log.domain.Jwlog;
/**
* <pre>
* 日志存储服务接口类
* </pre>
*
* @author chijiaqi
* @version 1.00.00
*
*/
public interface ILogStoreService {
/**
* 保存日志
*
* @param jwlog
*/
public abstract void saveSqlLogs(Jwlog jwlog);
}
日志存储服务实现类
package cn.edu.nfu.jw.log.service.imple;
import cn.edu.nfu.jw.log.dao.JwlogDao;
import cn.edu.nfu.jw.log.domain.Jwlog;
import cn.edu.nfu.jw.log.service.ILogStoreService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
/**
* @Auther: cjq
* @Date: 2020/4/15 11:33
* @Description:日志存储服务实现类
* @version:V1.0
*/
@Service
public class LogStoreServiceImpl implements ILogStoreService {
private static final Log log = LogFactory.getLog(LogStoreServiceImpl.class);
@Resource
protected ThreadPoolTaskExecutor logThreadPoolTaskExecutor;
@Resource
private JwlogDao<Jwlog> jwlogDao;
/*@Async("logThreadPoolTaskExecutor")
@Override
public void saveSqlLogs(Jwlog jwlog) {
try {
jwlogDao.save(jwlog);
} catch (Exception e) {
log.error("保存日志到数据库异常,具体异常信息为:" + e.getMessage(), e);
}
}*/
@Override
public void saveSqlLogs(Jwlog jwlog) {
try {
logThreadPoolTaskExecutor.execute(new SqlLogSaveServiceImpl(jwlog));
} catch (Exception e) {
log.error("保存日志异常,异常信息为:" + e.getMessage(), e);
}
}
/**
* @Auther: cjq
* @Date: 2020/4/15 11:33
* @Description:日志保存服务类
* @version:V1.0
*/
class SqlLogSaveServiceImpl implements Runnable {
private Jwlog jwlog;
public SqlLogSaveServiceImpl(Jwlog jwlog) {
super();
this.jwlog = jwlog;
}
public void run() {
try {
jwlogDao.save(jwlog);
} catch (Exception e) {
log.error("保存日志到数据库异常,具体异常信息为:" + e.getMessage(), e);
}
}
}
}
消费端接收消息
package cn.edu.nfu.jw.log.listener;
import cn.edu.nfu.jw.conf.RabbitConfig;
import cn.edu.nfu.jw.exception.ServiceException;
import cn.edu.nfu.jw.log.controller.JwlogController;
import cn.edu.nfu.jw.log.domain.Jwlog;
import cn.edu.nfu.jw.log.service.ILogStoreService;
import com.alibaba.fastjson.JSON;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
/**
* 日志消费
*/
@Component
@RabbitListener(queues = RabbitConfig.JW_QUEUE_DIRECT)//监听的队列名称 JwDirectQueue
public class DirectReceiver {
private static final Log log = LogFactory.getLog(JwlogController.class);
@Resource
private ILogStoreService logStoreService;
@RabbitHandler
public void process(String jsonStr) {
log.info("DirectReceiver消费者收到消息 : " + jsonStr);
try{
Jwlog jwlog = JSON.parseObject(jsonStr,Jwlog.class);
//通过线程池,开启多线程来保存日志
logStoreService.saveSqlLogs(jwlog);
}catch (Exception e){
log.error("从MQ队列里面获取信息保存异常,异常信息为:"+e.getMessage(),e);
throw new ServiceException(-1,"从MQ队列里面获取信息保存异常,异常信息为:"+e.getMessage());
}
}
}
补充记录问题
用上面线程池写日志,生产环境会出现cpu过高,内存溢出情况。经过增大jvm运行内存 -Xmx4g ,然后改用rabbitMq自带线程池处理后正常。下面贴出修改后的内容:
代码中使用@RabbitListener注解指定消费方法,默认情况是单线程监听队列,可以观察当队列有多个任务时消费
端每次只消费一个消息,单线程处理消息容易引起消息处理缓慢,消息堆积,不能最大利用硬件资源。
可以配置mq的容器工厂参数,增加并发处理数量即可实现多线程处理监听队列,实现多线程处理消息。
1、在RabbitmqConfig.java中添加容器工厂配置:
@Bean("customContainerFactory")
public SimpleRabbitListenerContainerFactory
containerFactory(SimpleRabbitListenerContainerFactoryConfigurer configurer, ConnectionFactory
connectionFactory) {
SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
factory.setConcurrentConsumers(DEFAULT_CONCURRENT);
factory.setMaxConcurrentConsumers(DEFAULT_CONCURRENT);
configurer.configure(factory, connectionFactory);
return factory;
}
2、在@RabbitListener注解中指定容器工厂
//改进后的消费者(可以设置多个节点,rabbitmq默认的负载均衡是轮询策略)
/**
* 日志消费
*/
@Component
@RabbitListener(queues = RabbitConfig.JW_QUEUE_DIRECT
,containerFactory = "customContainerFactory")//监听的队列名称 JwDirectQueue
public class DirectReceiver {
private static final Log log = LogFactory.getLog(JwlogController.class);
@Resource
private ILogStoreService logStoreService;
@Resource
private JwlogDao<Jwlog> jwlogDao;
@RabbitHandler
public void process(String jsonStr) {
log.info("DirectReceiver消费者收到消息 : " + jsonStr);
try{
Jwlog jwlog = JSON.parseObject(jsonStr,Jwlog.class);
//通过线程池,开启多线程来保存日志
jwlogDao.save(jwlog);
// logStoreService.saveSqlLogs(jwlog);
}catch (Exception e){
log.error("从MQ队列里面获取信息保存异常,异常信息为:"+e.getMessage(),e);
throw new ServiceException(-1,"从MQ队列里面获取信息保存异常,异常信息为:"+e.getMessage());
}
}
}
消费者增加手动应答
/**
* 预选消费
*/
@Component
@RabbitListener(queues = RabbitMqConstant.YX_QUEUE_DIRECT
, containerFactory = "xyContainerFactory")//监听的队列名称 XyDirectQueue
public class XyDirectReceiver {
private static final Log log = LogFactory.getLog(XyDirectReceiver.class);
@Resource
private CssYxkcService cssYxkcService;
@Resource(name = "xkRedisTemplate")
private RedisTemplate<String, String> redisTemplate;
// @RabbitHandler
public void process(String jsonStr, Channel channel, Message message) throws IOException {
log.info("XyDirectReceiver消费者收到消息 : " + jsonStr);
try {
CssYxStudent cssYxStudent = JSON.parseObject(jsonStr, CssYxStudent.class);
Boolean o = false;
if(cssYxStudent.getJxbid()==null){
o = cssYxkcService.saveYxkcInDatabase(cssYxStudent);
}else{
o = cssYxkcService.saveYxJxbInDatabase(cssYxStudent);
}
if (o) {//消费完成,删除标记
redisTemplate.delete("yxAsync:" + cssYxStudent.getPcid() + ":" + cssYxStudent.getXsid() + ":" + cssYxStudent.getKkkcid());
log.info( cssYxStudent.getXsid()+":预选保存成功!");
/*
*
* 确认一条消息:<br>
* channel.basicAck(deliveryTag, false); <br>
* deliveryTag:该消息的index <br>
* multiple:是否批量.true:将一次性ack所有小于deliveryTag的消息 <br>
*/
channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
log.info("XyDirectReceiver消费者消费消息 : " + jsonStr);
}else{
log.error("保存预选信息失败!");
}
} catch (Exception e) {
//消费者处理出了问题,需要告诉队列信息消费失败,失败后重新入队
/*
*
* 拒绝确认消息:<br>
* channel.basicNack(long deliveryTag, boolean multiple, boolean requeue) ; <br>
* deliveryTag:该消息的index<br>
* multiple:是否批量.true:将一次性拒绝所有小于deliveryTag的消息。<br>
* requeue:被拒绝的是否重新入队列 <br>
*/
channel.basicNack(message.getMessageProperties().getDeliveryTag(),
false, true);
log.error("保存预选信息,从MQ队列里面获取信息保存异常,异常信息为:" + e.getMessage(), e);
throw new ServiceException(-1, "保存预选信息,从MQ队列里面获取信息保存异常,异常信息为:" + e.getMessage());
}
}
}