RocketMQ順序消息消費
1. 應用場景
消息隊列中消息之間有先后的依賴關系，后一條消息的處理依賴于前一條消息的處理結果，那么比較適用于順序消息消費
2. 分析
RocketMQ
此圖可以看出Rocketmq中一個topic和它的mq之間的關系是一對多的關系，客戶端向broker中發送消息是根據topic發送的，而消費方消費時也是按照topic來消費的，那么我們怎么保證消息之間的順序性呢？首先，需要保證順序的消息要發送到同一個messagequeue中；其次，一個messagequeue只能被一個消費者消費，這點是由消息隊列的分配機制來保證的；最后，一個消費者內部對一個mq的消費要保證是有序的。
我們要做到生產者 - messagequeue - 消費者之間是一對一對一的關系。
3. 具體實現
3.1 消息生產者
Message message = new Message(topic, tags, key, body.getBytes());
boolean result;
try {
SendResult sendResult = getConfigBean().getMqProducer().send(message,queueSelector,args);
LoggerUtil.log(Level.INFO,"rocket message product result : " + sendResult.toString());
result = true;
}
catch (MQClientException e) {
throw new MQException("客戶端調用狀態異常", e);
}
catch (RemotingException | InterruptedException | MQBrokerException e) {
throw new MQException("遠程調用異常", e);
}
相比較普通消息的消費，順序消費在向broker發送消息的時候要指定MessageQueueSelector，此接口RocketMQ提供了三種實現SelectMessageQueueByRandoom，SelectMessageQueueByHash，SelectMessageQueueByMachineRoom，可由調用方自行根據業務實現。指定將消息發送到對應的隊列中去；
TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());
if (topicPublishInfo != null && topicPublishInfo.ok()) {
MessageQueue mq = null;
try {
mq = selector.select(topicPublishInfo.getMessageQueueList(), msg, arg);
}
catch (Throwable e) {
throw new MQClientException("select message queue throwed exception.", e);
}
if (mq != null) {
return this.sendKernelImpl(msg, mq, communicationMode, sendCallback);
}
else {
throw new MQClientException("select message queue return null.", null);
}
}
而普通消息的發送，客戶端調用方無需指定隊列，MQ會輪詢topic下面的MessageQueue發送消息，代碼如下
/**
* 如果lastBrokerName不為null，則尋找與其不同的MessageQueue
*/
public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
if (lastBrokerName != null) {
int index = this.sendWhichQueue.getAndIncrement();
for (int i = 0; i < this.messageQueueList.size(); i++) {
int pos = Math.abs(index++) % this.messageQueueList.size();
MessageQueue mq = this.messageQueueList.get(pos);
if (!mq.getBrokerName().equals(lastBrokerName)) {
return mq;
}
}
return null;
}
else {
int index = this.sendWhichQueue.getAndIncrement();
int pos = Math.abs(index) % this.messageQueueList.size();
return this.messageQueueList.get(pos);
}
}
3.2 消息消費者
消息消費方與普通消息消費只有一個地方不同,在consumer中注冊MessageListenerOrderly，而不是MessageListenerConcurrently
this.consumer.registerMessageListener((MessageListenerOrderly) (msgs, context) -> {
msgs.stream().forEach(
messageExt -> {
//TODO consume message
});
return ConsumeOrderlyStatus.SUCCESS;
});
我們進一步看下RocketMQ是怎么處理的ConsumeMessageOrderlyService在啟動的時候，如果是集群模式下會啟動一個單線程的定時調度任務，延遲一秒，時間間隔為20秒，執行rebalanceImpl的lockAll()方法。
public void start() {
// 啟動定時lock隊列服務
if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl
.messageModel())) {
this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
ConsumeMessageOrderlyService.this.lockMQPeriodically();
}
}, 1000 * 1, ProcessQueue.RebalanceLockInterval, TimeUnit.MILLISECONDS);
}
}
這個方法會鎖定相關broker下面的相關的messagequeue，對message對應的processQueue設置是否鎖定，這個地方下面會用到
class ConsumeRequest implements Runnable {
private final ProcessQueue processQueue;
private final MessageQueue messageQueue;
public ConsumeRequest(ProcessQueue processQueue, MessageQueue messageQueue) {
this.processQueue = processQueue;
this.messageQueue = messageQueue;
}
@Override
public void run() {
if (this.processQueue.isDroped()) {
log.warn("run, the message queue not be able to consume, because it's dropped. {}",
this.messageQueue);
return;
}
// 保證在當前Consumer內，同一隊列串行消費
final Object objLock = messageQueueLock.fetchLockObject(this.messageQueue);
synchronized (objLock) {
// 保證在Consumer集群，同一隊列串行消費
if (MessageModel.BROADCASTING
.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())
|| (this.processQueue.isLocked() && !this.processQueue.isLockExpired())) {
final long beginTime = System.currentTimeMillis();
for (boolean continueConsume = true; continueConsume;) {
if (this.processQueue.isDroped()) {
log.warn("the message queue not be able to consume, because it's dropped. {}",
this.messageQueue);
break;
}
if (MessageModel.CLUSTERING
.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl
.messageModel())
&& !this.processQueue.isLocked()) {
log.warn("the message queue not locked, so consume later, {}", this.messageQueue);
ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue,
this.processQueue, 10);
break;
}
if (MessageModel.CLUSTERING
.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl
.messageModel())
&& this.processQueue.isLockExpired()) {
log.warn("the message queue lock expired, so consume later, {}",
this.messageQueue);
ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue,
this.processQueue, 10);
break;
}
// 在線程數小于隊列數情況下，防止個別隊列被餓死
long interval = System.currentTimeMillis() - beginTime;
if (interval > MaxTimeConsumeContinuously) {
// 過10ms后再消費
ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue,
messageQueue, 10);
break;
}
final int consumeBatchSize =
ConsumeMessageOrderlyService.this.defaultMQPushConsumer
.getConsumeMessageBatchMaxSize();
List<MessageExt> msgs = this.processQueue.takeMessags(consumeBatchSize);
if (!msgs.isEmpty()) {
final ConsumeOrderlyContext context =
new ConsumeOrderlyContext(this.messageQueue);
ConsumeOrderlyStatus status = null;
// 執行Hook
ConsumeMessageContext consumeMessageContext = null;
if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) {
consumeMessageContext = new ConsumeMessageContext();
consumeMessageContext
.setConsumerGroup(ConsumeMessageOrderlyService.this.defaultMQPushConsumer
.getConsumerGroup());
consumeMessageContext.setMq(messageQueue);
consumeMessageContext.setMsgList(msgs);
consumeMessageContext.setSuccess(false);
ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl
.executeHookBefore(consumeMessageContext);
}
long beginTimestamp = System.currentTimeMillis();
//處理隊列加鎖
try {
this.processQueue.getLockConsume().lock();
if (this.processQueue.isDroped()) {
log.warn(
"consumeMessage, the message queue not be able to consume, because it's dropped. {}",
this.messageQueue);
break;
}
status =
messageListener.consumeMessage(Collections.unmodifiableList(msgs),
context);
}
catch (Throwable e) {
log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",//
RemotingHelper.exceptionSimpleDesc(e),//
ConsumeMessageOrderlyService.this.consumerGroup,//
msgs,//
messageQueue);
}
finally {
this.processQueue.getLockConsume().unlock();
}
// 針對異常返回代碼打印日志
if (null == status //
|| ConsumeOrderlyStatus.ROLLBACK == status//
|| ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT == status) {
log.warn("consumeMessage Orderly return not OK, Group: {} Msgs: {} MQ: {}",//
ConsumeMessageOrderlyService.this.consumerGroup,//
msgs,//
messageQueue);
}
long consumeRT = System.currentTimeMillis() - beginTimestamp;
// 用戶拋出異常或者返回null，都掛起隊列
if (null == status) {
status = ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
}
// 執行Hook
if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) {
consumeMessageContext.setStatus(status.toString());
consumeMessageContext.setSuccess(ConsumeOrderlyStatus.SUCCESS == status
|| ConsumeOrderlyStatus.COMMIT == status);
ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl
.executeHookAfter(consumeMessageContext);
}
// 記錄統計信息
ConsumeMessageOrderlyService.this.getConsumerStatsManager().incConsumeRT(
ConsumeMessageOrderlyService.this.consumerGroup, messageQueue.getTopic(),
consumeRT);
continueConsume =
ConsumeMessageOrderlyService.this.processConsumeResult(msgs, status,
context, this);
}
else {
continueConsume = false;
}
}
}
// 沒有拿到當前隊列的鎖，稍后再消費
else {
if (this.processQueue.isDroped()) {
log.warn("the message queue not be able to consume, because it's dropped. {}",
this.messageQueue);
return;
}
ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue,
this.processQueue, 100);
}
}
}
public ProcessQueue getProcessQueue() {
return processQueue;
}
public MessageQueue getMessageQueue() {
return messageQueue;
}
}
