Skip to content

[ISSUE #10577] Keep reentrant POP receives broker sticky #10652

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Aias00 wants to merge 1 commit into
base: develop
Choose a base branch
from
+97 −16
Open

[ISSUE #10577] Keep reentrant POP receives broker sticky #10652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,9 @@ public void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request,
CompletableFuture<PopResult> popFuture = this.messagingProcessor.popMessage(
ctx,
new ReceiveMessageQueueSelector(
request.getMessageQueue().getBroker().getName()
request.getMessageQueue().getBroker().getName(),
// Reentrant FIFO POP must stay on the original broker instead of falling back.
fifo && request.hasAttemptId()
),
group,
topic,
Expand Down Expand Up @@ -216,9 +218,19 @@ protected ReceiveMessageResponseStreamWriter createWriter(ProxyContext ctx,
protected static class ReceiveMessageQueueSelector implements QueueSelector {

private final String brokerName;
private final boolean brokerSticky;

public ReceiveMessageQueueSelector(String brokerName) {
this(brokerName, false);
}

/**
* @param brokerSticky if true, keep the target broker sticky and return null when the
* target broker is unavailable instead of falling back to another broker.
*/
public ReceiveMessageQueueSelector(String brokerName, boolean brokerSticky) {
this.brokerName = brokerName;
this.brokerSticky = brokerSticky;
}

@Override
Expand All @@ -229,6 +241,9 @@ public AddressableMessageQueue select(ProxyContext ctx, MessageQueueView message

if (StringUtils.isNotBlank(brokerName)) {
addressableMessageQueue = messageQueueSelector.getQueueByBrokerName(brokerName);
if (addressableMessageQueue != null || brokerSticky) {
return addressableMessageQueue;
}
}

if (addressableMessageQueue == null) {
Expand Down
96 changes: 81 additions & 15 deletions .../src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.rocketmq.proxy.grpc.v2.consumer;

import apache.rocketmq.v2.Code;
import apache.rocketmq.v2.Broker;
import apache.rocketmq.v2.ClientType;
import apache.rocketmq.v2.FilterExpression;
import apache.rocketmq.v2.FilterType;
Expand All @@ -26,6 +27,7 @@
import apache.rocketmq.v2.ReceiveMessageResponse;
import apache.rocketmq.v2.Resource;
import apache.rocketmq.v2.Settings;
import apache.rocketmq.v2.Subscription;
import com.google.protobuf.Duration;
import com.google.protobuf.util.Durations;
import io.grpc.stub.ServerCallStreamObserver;
Expand All @@ -50,6 +52,7 @@
import org.apache.rocketmq.proxy.common.ProxyContext;
import org.apache.rocketmq.proxy.config.ConfigurationManager;
import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest;
import org.apache.rocketmq.proxy.processor.QueueSelector;
import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue;
import org.apache.rocketmq.proxy.service.route.MessageQueueView;
import org.apache.rocketmq.remoting.protocol.route.BrokerData;
Expand All @@ -61,6 +64,8 @@

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
Expand Down Expand Up @@ -422,6 +427,53 @@ public void testReceiveMessage() {
assertEquals(Code.MESSAGE_NOT_FOUND, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues()));
}

@Test
public void testReceiveFifoMessageWithAttemptIdUsesStickyBrokerSelector() throws Exception {
StreamObserver<ReceiveMessageResponse> receiveStreamObserver = mock(ServerCallStreamObserver.class);
doNothing().when(receiveStreamObserver).onNext(any());

when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder()
.setSubscription(Subscription.newBuilder().setFifo(true).build())
.build());

PopResult popResult = new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList());
ArgumentCaptor<QueueSelector> queueSelectorCaptor = ArgumentCaptor.forClass(QueueSelector.class);
when(this.messagingProcessor.popMessage(
any(),
queueSelectorCaptor.capture(),
anyString(),
anyString(),
anyInt(),
anyLong(),
anyLong(),
anyInt(),
any(),
eq(true),
any(),
eq("attempt-id"),
anyLong())).thenReturn(CompletableFuture.completedFuture(popResult));

this.receiveMessageActivity.receiveMessage(
createContext(),
ReceiveMessageRequest.newBuilder()
.setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build())
.setMessageQueue(MessageQueue.newBuilder()
.setTopic(Resource.newBuilder().setName(TOPIC).build())
.setBroker(Broker.newBuilder().setName("missing-broker").build())
.build())
.setAttemptId("attempt-id")
.setAutoRenew(true)
.setFilterExpression(FilterExpression.newBuilder()
.setType(FilterType.TAG)
.setExpression("*")
.build())
.build(),
receiveStreamObserver
);

assertNull(queueSelectorCaptor.getValue().select(ProxyContext.create(), createMessageQueueView()));
}

private Code getResponseCodeFromReceiveMessageResponseList(List<ReceiveMessageResponse> responseList) {
for (ReceiveMessageResponse response : responseList) {
if (response.hasStatus()) {
Expand All @@ -433,6 +485,34 @@ private Code getResponseCodeFromReceiveMessageResponseList(List<ReceiveMessageRe

@Test
public void testReceiveMessageQueueSelector() throws Exception {
MessageQueueView messageQueueView = createMessageQueueView();
ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector("");

AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);
AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);
AddressableMessageQueue thirdSelect = selector.select(ProxyContext.create(), messageQueueView);

assertEquals(firstSelect, thirdSelect);
assertNotEquals(firstSelect, secondSelect);

for (int i = 0; i < 2; i++) {
ReceiveMessageActivity.ReceiveMessageQueueSelector selectorBrokerName =
new ReceiveMessageActivity.ReceiveMessageQueueSelector(BROKER_NAME + i);
assertEquals(BROKER_NAME + i, selectorBrokerName.select(ProxyContext.create(), messageQueueView).getBrokerName());
}

ReceiveMessageActivity.ReceiveMessageQueueSelector nonStickyMissingBroker =
new ReceiveMessageActivity.ReceiveMessageQueueSelector("missing-broker", false);
AddressableMessageQueue fallbackQueue = nonStickyMissingBroker.select(ProxyContext.create(), messageQueueView);
assertNotNull(fallbackQueue);
assertNotEquals("missing-broker", fallbackQueue.getBrokerName());

ReceiveMessageActivity.ReceiveMessageQueueSelector stickyMissingBroker =
new ReceiveMessageActivity.ReceiveMessageQueueSelector("missing-broker", true);
assertNull(stickyMissingBroker.select(ProxyContext.create(), messageQueueView));
}

private MessageQueueView createMessageQueueView() throws Exception {
TopicRouteData topicRouteData = new TopicRouteData();
List<QueueData> queueDatas = new ArrayList<>();
for (int i = 0; i < 2; i++) {
Expand All @@ -456,20 +536,6 @@ public void testReceiveMessageQueueSelector() throws Exception {
}
topicRouteData.setBrokerDatas(brokerDatas);

MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null);
ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector("");

AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView);
AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView);
AddressableMessageQueue thirdSelect = selector.select(ProxyContext.create(), messageQueueView);

assertEquals(firstSelect, thirdSelect);
assertNotEquals(firstSelect, secondSelect);

for (int i = 0; i < 2; i++) {
ReceiveMessageActivity.ReceiveMessageQueueSelector selectorBrokerName =
new ReceiveMessageActivity.ReceiveMessageQueueSelector(BROKER_NAME + i);
assertEquals(BROKER_NAME + i, selectorBrokerName.select(ProxyContext.create(), messageQueueView).getBrokerName());
}
return new MessageQueueView(TOPIC, topicRouteData, null);
}
}
Loading