环境
- windows10
- 1台NameServer和1台Broker的最简单环境
- JDK8
- RocketMQ 4.10.0
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>4.3.0</version>
</dependency>
通用代码块
使用Junit的单元测试 保证每次启动连接NameServer 和运行结束之后关闭Producer
@Slf4j
public class SimpleProducerTest {
private DefaultMQProducer producer;
private final String nameSrvAddr = "localhost:9876";
@Before
public void before() throws MQClientException {
producer = new DefaultMQProducer("test_group");
producer.setNamesrvAddr(nameSrvAddr);
producer.start();
}
@After
public void after() {
producer.shutdown();
}
}
发送同步消息
对于数据一致性要求较高的场景可以使用发送同步消息,同步发送给MQ之后,一定会等到MQ返回响应之后才能继续往下执行 核心API producer.send(msg) 返回SendResult结果包含了发送是否成功,消息ID等信息
/**
* 发送同步消息
*
* @throws Exception
*/
@Test
public void testSynchronouslyMsg() throws Exception {
String topic = "test_test";
String tag = "tag1";
Message msg = new Message(topic, tag, "testSimple1".getBytes());
SendResult result = producer.send(msg);
log.info(result.toString());
assertThat(result, Matchers.notNullValue());
}
发送异步消息
用于异步发送消息请求,传递回调函数用于管理消息处理之后的业务逻辑 producer.send(msg,callback)
@Test
public void testAsynchronouslyMsg() throws Exception {
String topic = "test_test";
String tag = "tag2";
Message msg = new Message(topic, tag, "testSimple2".getBytes());
producer.send(msg, new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
log.info(sendResult.toString());
assertThat(sendResult, Matchers.notNullValue());
}
@Override
public void onException(Throwable throwable) {
log.error(throwable.getMessage());
}
});
TimeUnit.SECONDS.sleep(3);
}
发送oneway消息
发送oneway消息没有返回值,在消息特别不敏感的情况下可以使用,一般用于日志记录等功能核心API
sendOneway(msg)
/**
* 发送oneway消息 发送之后没有返回值 用于发送一些可靠性要求不高的消息 如日志
* @throws Exception
*/
@Test
public void testOneWay() throws Exception {
String topic = "test_test";
String tag = "tag3";
Message msg = new Message(topic, tag, "testSimple3".getBytes());
producer.sendOneway(msg);
}
以上就简单消息的消费者代码示例
核心API subscribe(group,tag)订阅某一个消息group和tag
registerMessageListener(messageListener)添加消息监听的处理逻辑
@Slf4j
public class SimpleConsumerDemo {
public static void main(String[] args) throws MQClientException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("test_group");
consumer.setNamesrvAddr("localhost:9876");
consumer.subscribe("test_test", "*");
consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
msgs.forEach(msg -> {
log.info(msg.toString());
log.info(new String(msg.getBody()));
});
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
});
consumer.start();
System.out.println("consumer has started");
}
}
广播消息
生产者 与普通消息基本相同 核心逻辑在消费者
package rocket;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* @author : Jim Wu
* @version 1.0
* @function :
* @since : 2021/1/25 16:21
*/
@Slf4j
public class BroadcastProducerTest {
private DefaultMQProducer producer;
@Before
public void before() throws MQClientException {
producer = new DefaultMQProducer("group1");
producer.setNamesrvAddr("localhost:9876");
producer.start();
}
@Test
public void testBroadcast() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
for (int i = 0; i < 5; i++) {
Message msg = new Message("testBroadcast", "tag1", ("hello broadcast " + i).getBytes());
SendResult result = producer.send(msg);
log.info(result.toString());
}
}
@After
public void after() {
producer.shutdown();
}
}
消费者
核心修改 consumer.setMessageModel(MessageModel.BROADCASTING)将消费改为广播模式
package com.corn.rokectmq.tutorial;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function : 广播消费者
* @since : 2021/1/25 16:25
*/
public class BroadcastConsumerDemo {
public static void main(String[] args) throws MQClientException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
// 设置消费模式为广播消费
consumer.setMessageModel(MessageModel.BROADCASTING);
consumer.subscribe("testBroadcast", "*");
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.println(new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.println("consumer has started");
}
}
顺序消息
生产者
核心在于send的API 第二个参数传入MessageQueueSelector的实现类,表示根据某一种规则来选择topic下的Queue队列,这里的规则直接决定了MQ选择哪一个队列消费这条消息
package rocket;
import lombok.SneakyThrows;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function : 顺序消息
* @since : 2021/1/25 10:15
*/
public class OrderedProducerTest {
private DefaultMQProducer producer;
@SneakyThrows
@Before
public void before() {
producer = new DefaultMQProducer();
producer.setProducerGroup("group1");
producer.setNamesrvAddr("localhost:9876");
producer.start();
}
@SneakyThrows
@Test
public void orderMessageTest() {
String[] tags = new String[]{"TagA", "TagB", "TagC", "TagD", "TagE"};
for (int i = 0; i < 20; i++) {
int orderId = i;
Message msg = new Message();
msg.setTags(tags[orderId % tags.length]);
String body = "hello rocket" + i;
msg.setTopic("testOrder");
msg.setBody(body.getBytes());
producer.send(msg, new MessageQueueSelector() {
@Override
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
int index = orderId % mqs.size();
return mqs.get(index);
}
}, i);
}
}
@After
public void after() {
producer.shutdown();
}
}
消费者 无变动
package com.corn.rokectmq.tutorial;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function : 顺序消费demo
* @since : 2021/1/25 10:32
*/
public class OrderedConsumerDemo {
public static void main(String[] args) throws Exception {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
consumer.subscribe("testOrder", "TagA || TagD || TagE");
consumer.registerMessageListener(new MessageListenerOrderly() {
@Override
public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {
for (MessageExt msg : msgs) {
System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msg.getTags() + " " + new String(msg.getBody()) + "%n");
}
return ConsumeOrderlyStatus.SUCCESS;
}
});
consumer.start();
System.out.println("consumer has started");
}
}
延时消息
生产者
核心API在message中设置延迟时间
可选的延迟Level列表 messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
msg.setDelayTimeLevel(2);
package rocket;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* @author : Jim Wu
* @version 1.0
* @function :
* @since : 2021/1/25 16:49
*/
@Slf4j
public class ScheduledProducerTest {
private DefaultMQProducer producer;
@Before
public void before() throws MQClientException {
producer = new DefaultMQProducer("group1");
producer.setNamesrvAddr("localhost:9876");
producer.start();
}
@Test
public void testScheduled() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
for (int i = 0; i < 5; i++) {
byte[] body = ("hello scheduled " + i).getBytes();
Message msg = new Message("testScheduled", "tag1", body);
// messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
msg.setDelayTimeLevel(2);
SendResult result = producer.send(msg);
log.info(result.toString());
}
}
@After
public void after() {
producer.shutdown();
}
}
消费者
msg.getStoreTimestamp()API 可以拿到消费被存储的时间 不一定完全精准
package com.corn.rokectmq.tutorial;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function : 消费延迟消息
* @since : 2021/1/25 16:46
*/
public class ScheduledConsumerDemo {
public static void main(String[] args) throws MQClientException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
consumer.subscribe("testScheduled", "*");
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
// getStoreTimestamp 获取存储时间
System.out.println("Receive message[msgId=" + msg.getMsgId() + "] "
+ (System.currentTimeMillis() - msg.getStoreTimestamp()) + "ms later");
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.println("consumer has started");
}
}
批量消息
核心在于可以send可以传入一个Message集合
需要注意的是消息大小不能超过1MB 如果超过1MB需要对消息进行切分
切分逻辑参考官网 http://rocketmq.apache.org/docs/batch-example/
package rocket;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function : 批量发送消息 要求消息大小必须小于1M
* @since : 2021/1/25 19:13
*/
public class BatchMessageProducerTest {
private DefaultMQProducer producer;
private final String nameSrvAddr = "localhost:9876";
@Before
public void before() throws MQClientException {
producer = new DefaultMQProducer("group1");
producer.setNamesrvAddr(nameSrvAddr);
producer.start();
}
@After
public void after() {
producer.shutdown();
}
@Test
public void testBatchMessage() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
List<Message> msgs = new ArrayList<>();
for (int i = 0; i < 3; i++) {
Message m = new Message("testBatch", "tag1", ("hello Batch Message" + i).getBytes());
msgs.add(m);
}
producer.send(msgs);
}
}
消费者
package com.corn.rokectmq.tutorial;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function :
* @since : 2021/1/25 19:16
*/
public class BatchMessageConsumerDemo {
public static void main(String[] args) throws MQClientException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
// 设置消费模式为广播消费
consumer.subscribe("testBatch", "*");
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.println(new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.println("consumer has started");
}
}
过滤消息
生产者
核心在于message.putUserProperty(key,value) 可以给消息上添加一些属性
在消费端的时候可以根据这些属性 类似SQL的方法进行过滤
package rocket;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function :
* @since : 2021/1/25 19:18
*/
public class FilterProducerTest {
private DefaultMQProducer producer;
private final String nameSrvAddr = "localhost:9876";
@Before
public void before() throws MQClientException {
producer = new DefaultMQProducer("group1");
producer.setNamesrvAddr(nameSrvAddr);
producer.start();
}
@After
public void after() {
producer.shutdown();
}
@Test
public void testFilter() throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
for (int i = 0; i < 10; i++) {
Message m = new Message("testFilter", "tag1", ("hello Batch Message" + i).getBytes());
m.putUserProperty("a", String.valueOf(i));
producer.send(m);
}
}
}
消费者
consumer.subscribe(“testFilter”, MessageSelector.bySql(“a between 4 and 8”));
可以通过Message.Selector.bySql的方式写类似的SLQ92的语法规范去过滤消息
package com.corn.rokectmq.tutorial;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.MessageSelector;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function :
* @since : 2021/1/25 19:20
*/
public class FilterConsumerDemo {
public static void main(String[] args) throws MQClientException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
consumer.subscribe("testFilter", MessageSelector.bySql("a between 4 and 8"));
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.println(new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.println("consumer has started");
}
}
事务消息
MQ发送消息会先缓存在MQ中,调用本地事务的回调方法,根据状态如果是成功才会真正的发送给消费者,如果是回滚会删除这条消息,如果状态不确定会进入回查事务的回调函数继续确定消息的事务状态
类似数据库事务,可以用于做一些分布式事务实现的基础,在一些支付场景如用户A转账给用户B,如果是同步消息直接发给MQ,MQ返回成功,此时用户A的转账行为出现了异常数据库回滚了,但是消息已经发出去被其他服务消费了,导致A的钱没扣,但是B的钱多了造成生产事故
事务消息的原理
- Producer给Broker投递事务消息,带有唯一的key作为参数并保证幂等性
- Broker预提交先接受消息在本地存储起来,此时消息对Consumer不可见
- Broker预提交成功之后回调Producer的executeLocalTransaction回调本地事务,此时可以看做真正在数据中进行update的操作,如果update成功返回状态成功,失败则回滚消息,未知状态则回查事务
- 当Broker监听到Producer的Commit反馈,则真正的将消息投递给队列中,此时事务消息对Consumer可见,最终消息投递成功,事务结束
- 如果Broker监听到Producer的Rollback反馈,会回顾本地预提交的消息,最终消息删除不会投递给Consumer
- 如果超时未接口到Produer的反馈或者状态为未知,会定时回调checkLocalTranaction回查本地事务,再次根据Producer自己的执行情况ACK给Broker
生产者
API核心
- producer.sendMessageInTransaction(msg, null); 发送事务消息
- producer.setTransactionListener 添加事务监听器
- 实现监听器的2个核心方法
- executeLocalTransaction 执行本地的事务 如果LocalTransactionState.COMMIT_MESSAGE会发送消息给MQ,ROLLBACK_MESSAGE会回滚消息,UNKNOW会调用回查事务的回调函数
- checkLocalTransaction事务回查逻辑,在MQ预消息发现状态为UNKNOW的时候会调用该函数进行消息状态的回查,可以在这里处理一些回滚,重新提交的业务逻辑
package rocket;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.LocalTransactionState;
import org.apache.rocketmq.client.producer.TransactionListener;
import org.apache.rocketmq.client.producer.TransactionMQProducer;
import org.apache.rocketmq.client.producer.TransactionSendResult;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author : Jim Wu
* @version 1.0
* @function : 事务消息支持
* @since : 2021/1/26 10:07
*/
@Slf4j
public class TransactionProducerTest {
private TransactionMQProducer producer;
private AtomicInteger transactionIndex = new AtomicInteger(0);
private ConcurrentHashMap<String, Integer> localTrans = new ConcurrentHashMap<>();
@Before
public void before() throws MQClientException {
producer = new TransactionMQProducer("group1");
producer.setNamesrvAddr("localhost:9876");
producer.setTransactionListener(new TransactionListener() {
/**
* 执行本地事务
* @param msg
* @param arg
* @return
*/
@Override
public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {
int value = transactionIndex.getAndIncrement();
int status = value % 3;
localTrans.put(msg.getTransactionId(), status);
return LocalTransactionState.UNKNOW;
}
/**
* 回查事务逻辑
* @param msg
* @return
*/
@Override
public LocalTransactionState checkLocalTransaction(MessageExt msg) {
Integer status = localTrans.get(msg.getTransactionId());
System.out.println(status);
if (null != status) {
switch (status) {
case 0:
return LocalTransactionState.UNKNOW;
case 1:
return LocalTransactionState.COMMIT_MESSAGE;
case 2:
return LocalTransactionState.ROLLBACK_MESSAGE;
}
}
return LocalTransactionState.COMMIT_MESSAGE;
}
});
producer.start();
}
@After
public void after() {
producer.shutdown();
}
@Test
public void testTransaction() throws Exception {
String[] tags = new String[]{"TagA", "TagB", "TagC", "TagD", "TagE"};
for (int i = 0; i < 10; i++) {
Message msg = new Message("testTransaction", tags[i % tags.length], ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
// 发送事务消息 sendMessageInTransaction(消息, 携带的参数)
TransactionSendResult result = producer.sendMessageInTransaction(msg, null);
log.info(result.toString());
}
TimeUnit.MINUTES.sleep(1);
}
}
消费者
package com.corn.rokectmq.tutorial;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import java.util.List;
/**
* @author : Jim Wu
* @version 1.0
* @function :
* @since : 2021/1/26 11:59
*/
public class TransactionConsumerDemo {
public static void main(String[] args) throws MQClientException {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group1");
consumer.setNamesrvAddr("localhost:9876");
consumer.subscribe("testTransaction", "*");
consumer.registerMessageListener(new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
for (MessageExt msg : msgs) {
System.out.println(new String(msg.getBody()));
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
System.out.println("consumer has started");
}
}