老系统接入rabbitMq

1.rabbitMQ基础知识：https://www.rabbitmq.com

2. rabbitMQ安装：

2.1. mq下载地址：https://www.rabbitmq.com/download.html

2.2. Erlang下载地址：http://www.erlang.org/downloads

2.3.后台管理安装：rabbitmq-plugins enable rabbitmq_management

2.4.默认后台路径：http://127.0.0.1:15672/ 

2.5.默认用户帐号密码：guest/guest

3.与系统整合

3.1.系统用的是SpringMVC 4.1.X ，故POM文件选型如下：

        <dependency>
			<groupId>com.rabbitmq</groupId>
			<artifactId>amqp-client</artifactId>
			<version>5.6.0</version>
		</dependency>
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-api</artifactId>
			<version>1.7.25</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.amqp</groupId>
			<artifactId>spring-rabbit</artifactId>
			<version>1.4.5.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.amqp</groupId>
			<artifactId>spring-amqp</artifactId>
			<version>1.4.5.RELEASE</version>
		</dependency>

3.2.JAVA配置

3.2.1 基础配置：

@Configuration
@PropertySources({
		@PropertySource(value = "classpath:properties/ease_rabbit.properties", ignoreResourceNotFound = true) })
public class RabbitConfig {

	@Value("${rabbit.host}")
	protected String host;

	@Value("${rabbit.port}")
	protected Integer port;

	@Value("${rabbit.username}")
	protected String username;

	@Value("${rabbit.password}")
	protected String password;

	@Value("${rabbit.virtualHost}")
	protected String virtualHost;
	
	@Value("${rabbit.channelCacheSize}")
	protected Integer channelCacheSize;
	
	@Value("${rabbit.connectionTimeout}")
	protected Integer connectionTimeout;
	
	@Value("${rabbit.retryTime}")
	protected Integer retryTime;
	
	@Value("${rabbit.exception.retryTime}")
	protected Integer excepRetryTime;

	@Value("${rabbit.exchangeType}")
	protected String exchangeType;

	@Value("${rabbit.exchange}")
	protected String exchange;

	@Value("${rabbit.tandem.queue}")
	protected String tandemQueue;

	@Value("${rabbit.tandem.routingKey}")
	protected String tandemRoutingKey;

	@Value("${rabbit.tandem.prefetchCount}")
	protected Integer tandemPrefetchCount;

	@Value("${rabbit.tandem.concurrentConsumers}")
	protected Integer tandemConcurrentConsumers;
	
	@Value("${rabbit.tandem.callback.queue}")
	protected String tandemCallbackQueue;
	
	@Value("${rabbit.tandem.callback.routingKey}")
	protected String tandemCallbackRoutingKey;

	@Value("${rabbit.tandem.callback.prefetchCount}")
	protected Integer tandemCallbackPrefetchCount;
	
	@Value("${rabbit.tandem.callback.concurrentConsumers}")
	protected Integer tandemCallbackConcurrentConsumers;
	
	
	// 配置交换机类型
	@Bean(name = "Exchange")
	public AbstractExchange Exchange() throws IOException {
		if (exchangeType.contains("F")) {
			FanoutExchange fanoutExchange = new FanoutExchange(exchange, true, false);
			System.out.println("ProducerExchange-Fanout 生产者：广播路由");
			return fanoutExchange;
		} else if (exchangeType.contains("T")) {
			TopicExchange topicExchange = new TopicExchange(exchange, true, false);
			System.out.println("ProducerExchange-Topic 生产者：多路广播路由");
			return topicExchange;
		} else {
			DirectExchange directExchange = new DirectExchange(exchange, true, false);
			System.out.println("ProducerExchange-Direct 生产者：直接路由");
			return directExchange;
		}
	}
	
	
	//Queue
	@Bean(name = "TandemQueue")
	public Queue TandemQueue() throws IOException {
		Queue queue = new Queue(tandemQueue, true, false, false);
		return queue;
	}
	
	 //Queue绑定
	@Bean(name = "TandemBinding")
	public Binding TandemBinding() {
		Binding binding = new Binding(tandemQueue, DestinationType.QUEUE, exchange, tandemRoutingKey, null);
		return binding;
	}
	

}

 3.2.2 生产者配置：

@Configuration
public class RabbitProducerConfig extends RabbitConfig{

	// 配置生产者ConnectionFactory，配置基础连接信息
	@Bean(name = "ConnectionFactoryProducer")
	public ConnectionFactory ConnectionFactoryProducer() throws IOException {
		CachingConnectionFactory connectionFactory = new CachingConnectionFactory(host);
		connectionFactory.setPort(port);
		connectionFactory.setUsername(username);
		connectionFactory.setPassword(password);
		connectionFactory.setVirtualHost(virtualHost);
		// 网络波动相关微调
		connectionFactory.setChannelCacheSize(channelCacheSize);
		connectionFactory.setConnectionTimeout(connectionTimeout);
		// ack返回操作 消息发送确认
		connectionFactory.setPublisherConfirms(true);
		return connectionFactory;
	}
	
	@Bean
	public RabbitAdmin RabbitAdminProducer() throws IOException {
		return new RabbitAdmin(ConnectionFactoryProducer());
	}
	
	// 配置生产者消息转换器
	@Bean(name = "jsonMessageConverter")
	public AbstractMessageConverter ProducerMessageConverter() {
		FastJsonMessageConverter jsonMessageConverter = new FastJsonMessageConverter();
		return jsonMessageConverter;
	}
	
	// 生产者发送确认
	@Bean(name = "ProducerConfirm")
	public ProducerConfirm ProducerConfirm(){
		ProducerConfirm producerConfirm = new ProducerConfirm(retryTime);
		return producerConfirm;
	}

}

@Configuration
public class TandemProducerConfig extends RabbitProducerConfig{

	// 配置生产者Template
	@Bean(name = "TandemProducerRabbitTemplate")
	public RabbitTemplate TandemProducerRabbitTemplate() throws IOException {
		RabbitTemplate rabbitTemplate = new RabbitTemplate();
		rabbitTemplate.setConnectionFactory(ConnectionFactoryProducer());
		rabbitTemplate.setExchange(exchange);
		// 配置生产者信息转换器，封装发送信息的格式
		rabbitTemplate.setMessageConverter(ProducerMessageConverter());
		// ack返回操作
		rabbitTemplate.setMandatory(true);
		rabbitTemplate.setConfirmCallback(ProducerConfirm());
		
		//配置重试机制
		//指定延迟的倍数，比如delay=5000l,multiplier=2时，第一次重试为5秒后，第二次为10秒，第三次为20秒
        RetryTemplate retryTemplate = new RetryTemplate();
        ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
        backOffPolicy.setInitialInterval(500);
        backOffPolicy.setMultiplier(2.0); 
        backOffPolicy.setMaxInterval(10000);
        retryTemplate.setBackOffPolicy(backOffPolicy);
        rabbitTemplate.setRetryTemplate(retryTemplate);
		return rabbitTemplate;
	}
	
	// 实例化生产者代码，配置生产者要投递信息的RoutingKey
	@Bean(name = "TandemProducer")
	public TandemProducer TandemProducer() throws Exception {
		TandemProducer producer = new TandemProducer(tandemRoutingKey,excepRetryTime,TandemProducerRabbitTemplate());// t.test
		return producer;
	}

}

 

 3.2.3 消费者配置：

@Configuration
public class RabbitConsumerConfig extends RabbitConfig {

	@Bean(name = "ConnectionFactoryConsumer")
	public ConnectionFactory ConnectionFactoryConsumer() throws IOException {
		CachingConnectionFactory connectionFactory = new CachingConnectionFactory(host);
		connectionFactory.setPort(port);
		connectionFactory.setUsername(username);
		connectionFactory.setPassword(password);
		connectionFactory.setVirtualHost(virtualHost);
		// 网络波动相关微调
		connectionFactory.setChannelCacheSize(channelCacheSize);
		connectionFactory.setConnectionTimeout(connectionTimeout);
		return connectionFactory;
	}

	@Bean
	public RabbitAdmin RabbitAdminConsumer() throws IOException {
		return new RabbitAdmin(ConnectionFactoryConsumer());
	}
}

@Configuration
public class TandemConsumerConfig extends RabbitConsumerConfig{

	//配置消费者Template
	@Bean(name = "TandemConsumerRabbitTemplate")
	public RabbitTemplate TandemConsumerRabbitTemplate() throws IOException {
		RabbitTemplate rabbitTemplate = new RabbitTemplate(ConnectionFactoryConsumer());
		rabbitTemplate.setExchange(exchange);
		
		
		//配置重试机制
		//指定延迟的倍数，比如delay=5000l,multiplier=2时，第一次重试为5秒后，第二次为10秒，第三次为20秒
        RetryTemplate retryTemplate = new RetryTemplate();
        ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
        backOffPolicy.setInitialInterval(500);
        backOffPolicy.setMultiplier(2.0); 
        backOffPolicy.setMaxInterval(10000);
        retryTemplate.setBackOffPolicy(backOffPolicy);
        rabbitTemplate.setRetryTemplate(retryTemplate);
        
		return rabbitTemplate;
	}

	//实例化消费者代码
	@Bean(name = "TandemConsumer")
	public TandemConsumer TandemConsumer() {
		TandemConsumer consumer = new TandemConsumer();
		return consumer;
	}
	
	//配置监听模式，当有消息到达时会通知监听在对应的队列上的监听对象
	@Bean(name = "TandemListenerContainer")
	public AbstractMessageListenerContainer TandemListenerContainer() throws IOException {
		SimpleMessageListenerContainer simpleMessageListenerContainer = new SimpleMessageListenerContainer();
		simpleMessageListenerContainer.setConnectionFactory(ConnectionFactoryConsumer());
		simpleMessageListenerContainer.setQueueNames(tandemQueue);
		if(tandemPrefetchCount!=null&&tandemPrefetchCount.intValue()>1) {
			simpleMessageListenerContainer.setPrefetchCount(tandemPrefetchCount);
		}
		simpleMessageListenerContainer.setConcurrentConsumers(tandemConcurrentConsumers);// 启动的线程数目
		simpleMessageListenerContainer.setMessageListener(TandemConsumer());
		
		// 手动确认消息返回
		simpleMessageListenerContainer.setAcknowledgeMode(AcknowledgeMode.MANUAL);
		return simpleMessageListenerContainer;
	}

}

 

4.共同API：

4.1.生产者API：

public class TandemProducer {

	private static Logger logger = LoggerFactory.getLogger(TandemProducer.class);

	private RabbitTemplate rabbitTemplate;

	@Autowired
	IFlowRabbitIdempotencyService flowRabbitIdempotencyService;

	private String routingKey;
	
	private int excepRetryTime;

	public TandemProducer() {
	}
	
	public TandemProducer(String routingKey, Integer excepRetryTime,RabbitTemplate rabbitTemplate) {
		this.routingKey = routingKey;
		this.excepRetryTime = excepRetryTime.intValue();
		this.rabbitTemplate = rabbitTemplate;
	}

	/**
	 * 发送mq
	 * 
	 * @param message
	 */
	public void sendDataToQueue(MessageInfo message) {
		
		long start = System.currentTimeMillis();
		logger.info("串接请求发送开始:{}", message);
		int retriedCnt = retrySend("", "", message, excepRetryTime, 0, false);
		logger.info("串接请求发送结束,消耗时间:{},重试次数:{}", System.currentTimeMillis()-start,retriedCnt);
	}
	
	/**
	 * 发送消息 实现重发机制
	 * @param rst
	 * @param id
	 * @param message
	 * @param retryCnt
	 * @param retriedCnt
	 * @param isSuccess
	 * @return
	 */
	private int retrySend(String rst, String id, MessageInfo message, int retryCnt, int retriedCnt, boolean isSuccess) {

		if (retriedCnt >= retryCnt || isSuccess) {
			return retriedCnt;
		}
		
		try {
			Thread.sleep(retriedCnt * 1000);
			retriedCnt++;

			if (RabbitConst.SUCCESS_STR.equals(rst)) {
				String[] idInfo = id.split(RabbitConst.DASH);
				if (idInfo == null || idInfo.length != 2) {
					logger.error("correlationDataId error :{}", id);
					return retriedCnt;
				}
				String type = idInfo[0];
				String uuid = idInfo[1];

				reSendDataToQueue(type, id, message);
				isSuccess = true;
			} else {
				Map<String, String> rstMap = flowRabbitIdempotencyService.insertTandemSendIdem(message, message.getParentInsId());
				rst = rstMap.get("rst");
				id = rstMap.get("id");
				if (RabbitConst.SUCCESS_STR.equals(rst)) {
					logger.info("串接插入幂等表成功:{}", id);
					// 发送rabbitmq
					message.setCorrelationDataId(id);
					rabbitTemplate.convertAndSend(this.routingKey, message, new CorrelationData(id));
					isSuccess = true;
				} else {
					logger.error("串接插入幂等表失败:{}", message);
					throw new Exception("串接插入幂等表失败");
				}
			}
		} catch (Exception e) {
			logger.error("maybe Rabbit server down");
			logger.error(e.getMessage(), e);
			isSuccess = false;
		}

		return retrySend(rst, id, message, retryCnt, retriedCnt, isSuccess);
	}

	/**
	 * 发送失败重新发送
	 * @param type
	 * @param uuid
	 * @param message
	 */
	public void reSendDataToQueue(String type, String correlationDataId, MessageInfo message) {

		long start = System.currentTimeMillis();
		logger.info("rabbitMQ producer reSend. type:{},correlationDataId{}, message:{}",type,correlationDataId,message);
		//MessageInfo messageInfo = JsonUtil.fromJson(message, MessageInfo.class);
		rabbitTemplate.convertAndSend(this.routingKey, message, new CorrelationData(correlationDataId));
		logger.info("reSendDataToQueue, spend{}", System.currentTimeMillis()-start);

	}

	public String getRoutingKey() {
		return routingKey;
	}

	public void setRoutingKey(String routingKey) {
		this.routingKey = routingKey;
	}

	public int getExcepRetryTime() {
		return excepRetryTime;
	}

	public void setExcepRetryTime(int excepRetryTime) {
		this.excepRetryTime = excepRetryTime;
	}

}

4.2.消费者API：

public class TandemConsumer implements ChannelAwareMessageListener {

	private static Logger logger = LoggerFactory.getLogger(TandemConsumer.class);

	@Autowired
	IFlowRabbitIdempotencyService flowRabbitIdempotencyService;

	public void onMessage(Message message, Channel channel) {

		long start = System.currentTimeMillis();
		
		try {
			// 手动确认消息返回
			channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
			
			String data = new String(message.getBody());
			logger.info("串接请求处理开始:{}", data);
			MQData mqData = JsonUtil.fromJson(data, MQData.class);
			MessageInfo messageInfo = mqData.getData();
			String id = messageInfo.getCorrelationDataId();
			String[] idInfo = id.split(RabbitConst.DASH);
			
			if(idInfo==null||idInfo.length != 2) {
				logger.error("correlationDataId error :{}",id);
				return;
			}
			
			String type = idInfo[0];
			String uuid = idInfo[1];
			
			// 幂等验证
			int rst  = flowRabbitIdempotencyService.checkIdem(uuid);
			if(rst==1) {
                // dosomething

                // 处理结果 更新
				flowRabbitIdempotencyService.finishIdem(uuid, processInsId);
			}else {
				logger.info("此请求已处理或处理中,uuid:{}",uuid);
			}
			
		} catch (Exception e) {
			logger.info("流程串接监听失败:{}",e.getMessage());
			logger.error(e.getMessage(),e);
		}
		
		logger.info("串接请求处理结束,消耗时间:{}", System.currentTimeMillis()-start);

	}
}

5.ack确认：

public class ProducerConfirm implements ConfirmCallback {

	private static Logger logger = LoggerFactory.getLogger(ProducerConfirm.class);

	@Lazy
	@Autowired
	TandemProducer tandemProducer;

	@Autowired
	IFlowRabbitIdempotencyService flowRabbitIdempotencyService;

	private Integer retryTime;

	public ProducerConfirm(Integer retryTime) {
		this.retryTime = retryTime;
	}

	@Override
	public void confirm(CorrelationData correlationData, boolean ack, String cause) {
		if (!ack) {
			logger.error("send message failed,id:{} ", correlationData.toString());
			
			long start = System.currentTimeMillis();
			logger.info("串接请求重新发送开始:{}", correlationData.toString());
			
			String id = correlationData.getId();
			String[] idInfo = id.split(RabbitConst.DASH);
			
			if(idInfo==null||idInfo.length != 2) {
				logger.error("correlationDataId error :{}",correlationData.toString());
				return;
			}
			
			String type = idInfo[0];
			String uuid = idInfo[1];
			
			// 根据重试次数查询幂等bean
			FlowRabbitIdempotencyTandem tandem = flowRabbitIdempotencyService.getFlowRabbitIdempotencyTandem(uuid,retryTime);
			if (tandem == null) {
				logger.error("had over error retry times. correlationDataId:{},times:{},",correlationData.toString(),retryTime);
				return;
			}
			
			int retryCnt = tandem.getRetryCnt().intValue();
			
			if (!(retryCnt < retryTime)) {
				logger.error("had over error retry times. correlationDataId:{},times:{},",correlationData.toString(),retryTime);
				return;
			}
			
			try {
				Thread.sleep(retryCnt*100);
			} catch (InterruptedException e) {
				logger.error(e.getMessage(),e);
			}
			
			String msgJson = JsonUtil.toJson(tandem.getMessage());
			MessageInfo messageInfo = JsonUtil.fromJson(msgJson, MessageInfo.class);
				tandemProducer.reSendDataToQueue(type, id, messageInfo);
			
			flowRabbitIdempotencyService.updRetryCnt(uuid, retryTime);
			logger.info("串接请求重新发送结束:{},重试次数:{}", System.currentTimeMillis()-start,retryCnt);
		}
	}

	public Integer getRetryTime() {
		return retryTime;
	}

	public void setRetryTime(Integer retryTime) {
		this.retryTime = retryTime;
	}

}

6.总结：

6.1.选型需要尽量用比较新的而且维护比较久的，向下兼容本系统，想上使用新的rabbitmq

6.2.需要处理网络原因导致的发送失败和ack返回失败的重发

6.3.根据5.2的处理，会导致一个处理发送多条的情况，需要幂等验证处理，保证不重复处理

6.4.网络断了重连之后，如果生产者和消费者使用同一个connection的话会造成Queue阻塞，所有生产者和消费者的连接需要分开

6.5.生产者和消费者会分开发布，所以生产者的配置和消费者的配置得分开

参考文档：

百度，google