springboot整合rabbitmq和ThreadPool实现异步调用

最新推荐文章于 2024-04-10 21:32:33 发布

Richard Chijq

最新推荐文章于 2024-04-10 21:32:33 发布

阅读量2.5k

点赞数 1

分类专栏： rabbitmq java 文章标签： java rabbitmq

本文链接：https://blog.csdn.net/qq837993702/article/details/105535750

版权

java 同时被 2 个专栏收录

16 篇文章 0 订阅

订阅专栏

rabbitmq

5 篇文章 0 订阅

订阅专栏

添加rabbitMq依赖

<dependency>
	<groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-amqp</artifactId>
</dependency>

配置rabbitMq 服务器

服务端和消费端都要配置

#配置rabbitMq 服务器
spring.rabbitmq.host= 10.10.240.xxx
spring.rabbitmq.port= 5672
spring.rabbitmq.username= chijiaqi
spring.rabbitmq.password= 123456

服务端配置类

package cn.edu.nfu.jw.conf;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.*;
import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;


/*
 * @Author : chijiaqi
 * @CreateTime : 2020/4/14
 * @Description :
*/
/**
 Broker:它提供一种传输服务,它的角色就是维护一条从生产者到消费者的路线，保证数据能按照指定的方式进行传输,
 Exchange：消息交换机,它指定消息按什么规则,路由到哪个队列。
 Queue:消息的载体,每个消息都会被投到一个或多个队列。
 Binding:绑定，它的作用就是把exchange和queue按照路由规则绑定起来.
 Routing Key:路由关键字,exchange根据这个关键字进行消息投递。
 vhost:虚拟主机,一个broker里可以有多个vhost，用作不同用户的权限分离。
 Producer:消息生产者,就是投递消息的程序.
 Consumer:消息消费者,就是接受消息的程序.
 Channel:消息通道,在客户端的每个连接里,可建立多个channel.
 */
@Configuration
public class RabbitConfig {

    private final Logger logger = LoggerFactory.getLogger(this.getClass());

    @Value("${spring.rabbitmq.host}")
    private String host;

    @Value("${spring.rabbitmq.port}")
    private int port;

    @Value("${spring.rabbitmq.username}")
    private String username;

    @Value("${spring.rabbitmq.password}")
    private String password;


    public static final String JW_EXCHANGE_DIRECT = "my-mq-exchange_A";


    public static final String JW_QUEUE_DIRECT = "JwDirectQueue";

    public static final String JW_ROUTINGKEY_DIRECT = "spring-boot-routingKey_A";

    @Bean
    public ConnectionFactory connectionFactory() {
        CachingConnectionFactory connectionFactory = new CachingConnectionFactory(host,port);
        connectionFactory.setUsername(username);
        connectionFactory.setPassword(password);
        connectionFactory.setVirtualHost("/");
        connectionFactory.setPublisherConfirms(true);
        return connectionFactory;
    }

    @Bean
    @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
    //必须是prototype类型
    public RabbitTemplate rabbitTemplate() {
        RabbitTemplate template = new RabbitTemplate(connectionFactory());
        return template;
    }


    /**
     * 针对消费者配置
     * 1. 设置交换机类型
     * 2. 将队列绑定到交换机
     FanoutExchange: 将消息分发到所有的绑定队列，无routingkey的概念
     HeadersExchange ：通过添加属性key-value匹配
     DirectExchange:按照routingkey分发到指定队列
     TopicExchange:多关键字匹配
     */
    @Bean
    public DirectExchange defaultExchange() {
        return new DirectExchange(JW_EXCHANGE_DIRECT);
    }
    /**
     * 获取队列A
     * @return
     */
    @Bean
    public Queue queueA() {
        return new Queue(JW_QUEUE_DIRECT, true); //队列持久
    }

    //一个交换机可以绑定多个消息队列，也就是消息通过一个交换机，可以分发到不同的队列当中去。

    @Bean
    public Binding binding() {
        return BindingBuilder.bind(queueA()).to(defaultExchange()).with(JW_ROUTINGKEY_DIRECT);
    }
}

服务端发送消息

@Autowired
RabbitTemplate rabbitTemplate;  //使用RabbitTemplate,这提供了接收/发送等等方法

rabbitTemplate.convertAndSend(RabbitConfig.JW_EXCHANGE_DIRECT, RabbitConfig.JW_ROUTINGKEY_DIRECT, JSON.toJSONString(jwlog));//jwlog为消息实体类

消费端配置线程池

package cn.edu.nfu.jw.log.conf;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;

/**
 * @Auther: cjq
 * @Date: 2020/4/15 11:33
 * @Description:
 * @version:V1.0
 * 用到线程池的时候还需要创建Executors，spring中有十分优秀的支持，
 * 就是注解@EnableAsync就可以使用多线程，@Async加在线程任务的方法上（需要异步执行的任务），
 * 定义一个线程任务，通过spring提供的ThreadPoolTaskExecutor就可以使用线程池
 *
 */
@Configuration
@EnableAsync
public class ThreadPoolConf {

    @Bean
    public ThreadPoolTaskExecutor logThreadPoolTaskExecutor() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        // 设置核心线程数，默认为1
        executor.setCorePoolSize(10);
        // 设置最大线程数，默认为Integer.MAX_VALUE
        executor.setMaxPoolSize(50);
        // 设置队列容量，一般需要设置值大于等于notifyScheduledMainExecutor.maxNum；默认为Integer.MAX_VALUE
//        executor.setQueueCapacity(2);
        // 设置线程池维护线程所允许的空闲时间，默认为60s
        executor.setKeepAliveSeconds(300);
        // 设置默认线程名称
        executor.setThreadNamePrefix("jw-thread-");
        // 设置拒绝策略rejection-policy：当pool已经达到max size的时候，如何处理新任务 CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        // 等待所有任务结束后再关闭线程池
        executor.setWaitForTasksToCompleteOnShutdown(true);
        return executor;
    }

}

线程池业务实现类

日志存储服务接口类

package cn.edu.nfu.jw.log.service;


import cn.edu.nfu.jw.log.domain.Jwlog;


/**
 * <pre>
 * 日志存储服务接口类
 * </pre>
 * 
 * @author chijiaqi
 * @version 1.00.00
 *
 */
public interface ILogStoreService {

	/**
	 * 保存日志
	 * 
	 * @param jwlog
	 */
	public abstract void saveSqlLogs(Jwlog jwlog);
}

日志存储服务实现类

package cn.edu.nfu.jw.log.service.imple;

import cn.edu.nfu.jw.log.dao.JwlogDao;
import cn.edu.nfu.jw.log.domain.Jwlog;
import cn.edu.nfu.jw.log.service.ILogStoreService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;


/**
 * @Auther: cjq
 * @Date: 2020/4/15 11:33
 * @Description:日志存储服务实现类
 * @version:V1.0
 */
@Service
public class LogStoreServiceImpl implements ILogStoreService {
    private static final Log log = LogFactory.getLog(LogStoreServiceImpl.class);

    @Resource
    protected ThreadPoolTaskExecutor logThreadPoolTaskExecutor;

    @Resource
    private JwlogDao<Jwlog> jwlogDao;

    /*@Async("logThreadPoolTaskExecutor")
    @Override
    public void saveSqlLogs(Jwlog jwlog) {
        try {
            jwlogDao.save(jwlog);
        } catch (Exception e) {
            log.error("保存日志到数据库异常，具体异常信息为：" + e.getMessage(), e);
        }
    }*/

    @Override
    public void saveSqlLogs(Jwlog jwlog) {
        try {
            logThreadPoolTaskExecutor.execute(new SqlLogSaveServiceImpl(jwlog));
        } catch (Exception e) {
            log.error("保存日志异常，异常信息为：" + e.getMessage(), e);
        }
    }


    /**
     * @Auther: cjq
     * @Date: 2020/4/15 11:33
     * @Description:日志保存服务类
     * @version:V1.0
     */
    class SqlLogSaveServiceImpl implements Runnable {
        private Jwlog jwlog;

        public SqlLogSaveServiceImpl(Jwlog jwlog) {
            super();
            this.jwlog = jwlog;
        }

        public void run() {
            try {
                jwlogDao.save(jwlog);
            } catch (Exception e) {
                log.error("保存日志到数据库异常，具体异常信息为：" + e.getMessage(), e);
            }
        }
    }
}

消费端接收消息

package cn.edu.nfu.jw.log.listener;

import cn.edu.nfu.jw.conf.RabbitConfig;
import cn.edu.nfu.jw.exception.ServiceException;
import cn.edu.nfu.jw.log.controller.JwlogController;
import cn.edu.nfu.jw.log.domain.Jwlog;
import cn.edu.nfu.jw.log.service.ILogStoreService;
import com.alibaba.fastjson.JSON;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;

/**
 * 日志消费
 */
@Component
@RabbitListener(queues = RabbitConfig.JW_QUEUE_DIRECT)//监听的队列名称 JwDirectQueue
public class DirectReceiver {
    private static final Log log = LogFactory.getLog(JwlogController.class);
    @Resource
    private ILogStoreService logStoreService;
    @RabbitHandler
    public void process(String jsonStr) {
        log.info("DirectReceiver消费者收到消息  : " + jsonStr);
        try{
            Jwlog jwlog = JSON.parseObject(jsonStr,Jwlog.class);
            //通过线程池，开启多线程来保存日志
            logStoreService.saveSqlLogs(jwlog);
        }catch (Exception e){
            log.error("从MQ队列里面获取信息保存异常，异常信息为："+e.getMessage(),e);
            throw new ServiceException(-1,"从MQ队列里面获取信息保存异常,异常信息为："+e.getMessage());
        }
    }
 
}

补充记录问题

用上面线程池写日志，生产环境会出现cpu过高，内存溢出情况。经过增大jvm运行内存 -Xmx4g ，然后改用rabbitMq自带线程池处理后正常。下面贴出修改后的内容：

代码中使用@RabbitListener注解指定消费方法，默认情况是单线程监听队列，可以观察当队列有多个任务时消费
端每次只消费一个消息，单线程处理消息容易引起消息处理缓慢，消息堆积，不能最大利用硬件资源。
可以配置mq的容器工厂参数，增加并发处理数量即可实现多线程处理监听队列，实现多线程处理消息。
1、在RabbitmqConfig.java中添加容器工厂配置：

@Bean("customContainerFactory")
    public SimpleRabbitListenerContainerFactory
    containerFactory(SimpleRabbitListenerContainerFactoryConfigurer configurer, ConnectionFactory
            connectionFactory) {
        SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
        factory.setConcurrentConsumers(DEFAULT_CONCURRENT);
        factory.setMaxConcurrentConsumers(DEFAULT_CONCURRENT);
        configurer.configure(factory, connectionFactory);
        return factory;
    }

2、在@RabbitListener注解中指定容器工厂
//改进后的消费者（可以设置多个节点，rabbitmq默认的负载均衡是轮询策略）

/**
 * 日志消费
 */
@Component
@RabbitListener(queues = RabbitConfig.JW_QUEUE_DIRECT
,containerFactory = "customContainerFactory")//监听的队列名称 JwDirectQueue
public class DirectReceiver {
    private static final Log log = LogFactory.getLog(JwlogController.class);
    @Resource
    private ILogStoreService logStoreService;
    @Resource
    private JwlogDao<Jwlog> jwlogDao;

    @RabbitHandler
    public void process(String jsonStr) {
        log.info("DirectReceiver消费者收到消息  : " + jsonStr);
        try{
            Jwlog jwlog = JSON.parseObject(jsonStr,Jwlog.class);
            //通过线程池，开启多线程来保存日志
            jwlogDao.save(jwlog);
//            logStoreService.saveSqlLogs(jwlog);
        }catch (Exception e){
            log.error("从MQ队列里面获取信息保存异常，异常信息为："+e.getMessage(),e);
            throw new ServiceException(-1,"从MQ队列里面获取信息保存异常,异常信息为："+e.getMessage());
        }
    }
 
}

消费者增加手动应答

/**
 * 预选消费
 */
@Component
@RabbitListener(queues = RabbitMqConstant.YX_QUEUE_DIRECT
        , containerFactory = "xyContainerFactory")//监听的队列名称 XyDirectQueue
public class XyDirectReceiver {
    private static final Log log = LogFactory.getLog(XyDirectReceiver.class);
    @Resource
    private CssYxkcService cssYxkcService;
    @Resource(name = "xkRedisTemplate")
    private RedisTemplate<String, String> redisTemplate;

//    @RabbitHandler
    public void process(String jsonStr, Channel channel, Message message) throws IOException {
        log.info("XyDirectReceiver消费者收到消息  : " + jsonStr);
        try {
            CssYxStudent cssYxStudent = JSON.parseObject(jsonStr, CssYxStudent.class);
            Boolean o = false;
            if(cssYxStudent.getJxbid()==null){
                o = cssYxkcService.saveYxkcInDatabase(cssYxStudent);
            }else{
                o = cssYxkcService.saveYxJxbInDatabase(cssYxStudent);
            }
            if (o) {//消费完成，删除标记
                redisTemplate.delete("yxAsync:" + cssYxStudent.getPcid() + ":" + cssYxStudent.getXsid() + ":" + cssYxStudent.getKkkcid());
                log.info( cssYxStudent.getXsid()+"：预选保存成功！");
/*
*
                 * 确认一条消息：<br>
                 * channel.basicAck(deliveryTag, false); <br>
                 * deliveryTag:该消息的index <br>
                 * multiple：是否批量.true:将一次性ack所有小于deliveryTag的消息 <br>

*/

                channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
                log.info("XyDirectReceiver消费者消费消息  : " + jsonStr);
            }else{
                log.error("保存预选信息失败！");
            }
        } catch (Exception e) {
            //消费者处理出了问题，需要告诉队列信息消费失败，失败后重新入队
/*
*
             * 拒绝确认消息:<br>
             * channel.basicNack(long deliveryTag, boolean multiple, boolean requeue) ; <br>
             * deliveryTag:该消息的index<br>
             * multiple：是否批量.true:将一次性拒绝所有小于deliveryTag的消息。<br>
             * requeue：被拒绝的是否重新入队列 <br>

*/
            channel.basicNack(message.getMessageProperties().getDeliveryTag(),
                    false, true);
            log.error("保存预选信息，从MQ队列里面获取信息保存异常，异常信息为：" + e.getMessage(), e);
            throw new ServiceException(-1, "保存预选信息，从MQ队列里面获取信息保存异常,异常信息为：" + e.getMessage());
        }
    }

}

Richard Chijq

关注

1
点赞
踩
14

收藏

觉得还不错? 一键收藏
打赏
2
评论
springboot整合rabbitmq和ThreadPool实现异步调用

添加rabbitMq依赖<dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-amqp</artifactId></dependency>配置rabbitMq 服务器服务端和消费端都要...
复制链接

扫一扫