From b84c1d45fe8eed117b447eb5660f608e89ef519a Mon Sep 17 00:00:00 2001 From: "dejing.liu" Date: Thu, 2 Jul 2026 23:36:05 +0800 Subject: [PATCH] [lb]: support tcp ipvs listener params Add dataPlane and forwardMode to TCP listener create and inventory paths so Cloud can select the IPVS data plane for L4 listeners. Reject forwardMode updates after listener creation and reject healthCheckTimeout for TCP IPVS listeners because the IPVS health daemon does not consume that HAProxy-style timeout. Regenerate the load balancer listener API doc templates so the API template check stays clean after docpremium generation. Test: mvn -pl plugin/loadBalancer -am -DskipTests install Test: ./runMavenProfile docpremium Resolves: ZSTAC-86152 Change-Id: I34d94c59651369e7de9d8ae9b33f7e6025300948 --- .../lb/APIChangeLoadBalancerListenerMsg.java | 11 + ...ngeLoadBalancerListenerMsgDoc_zh_cn.groovy | 10 + ...ateLoadBalancerListenerMsgDoc_zh_cn.groovy | 2 +- .../lb/LoadBalancerApiInterceptor.java | 55 +++-- .../network/service/lb/LoadBalancerBase.java | 12 +- .../lb/LoadBalancerListenerInventory.java | 54 +++++ .../sdk/ChangeLoadBalancerListenerAction.java | 3 + .../sdk/LoadBalancerListenerInventory.java | 16 ++ .../TcpIpvsLoadBalancerListenerApiCase.groovy | 209 +++++++++++++++--- .../CloudOperationsErrorCode.java | 20 ++ 10 files changed, 342 insertions(+), 50 deletions(-) diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java index 6e47b12bebe..7a4ae4622a2 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java @@ -95,6 +95,9 @@ public class APIChangeLoadBalancerListenerMsg extends APIMessage implements Load @APIParam(required = false) private List httpCompressAlgos; + @APIParam(validValues = {LoadBalancerConstants.FORWARD_MODE_FULL_NAT, LoadBalancerConstants.FORWARD_MODE_NAT, LoadBalancerConstants.FORWARD_MODE_DR}, required = false) + private String forwardMode; + @APINoSee private String loadBalancerUuid; @@ -318,6 +321,14 @@ public void setHttpCompressAlgos(List httpCompressAlgos) { this.httpCompressAlgos = httpCompressAlgos; } + public String getForwardMode() { + return forwardMode; + } + + public void setForwardMode(String forwardMode) { + this.forwardMode = forwardMode; + } + public static APIChangeLoadBalancerListenerMsg __example__() { APIChangeLoadBalancerListenerMsg msg = new APIChangeLoadBalancerListenerMsg(); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy index bf2c1f80054..f218b996f5c 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy @@ -141,6 +141,16 @@ doc { optional true since "3.9" } + column { + name "forwardMode" + enclosedIn "changeLoadBalancerListener" + desc "监听器使用 IPVS 数据平面时的转发模式。该参数只允许在创建监听器时指定，创建后不允许修改。" + location "body" + type "String" + optional true + since "5.5.28" + values ("full_nat","nat","dr") + } column { name "aclStatus" enclosedIn "changeLoadBalancerListener" diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy index 099aa923130..8e618f46cb6 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy @@ -238,7 +238,7 @@ doc { column { name "forwardMode" enclosedIn "params" - desc "监听器使用 IPVS 数据平面时的转发模式。当前 TCP IPVS 监听器支持 full_nat。" + desc "监听器使用 IPVS 数据平面时的转发模式。当前 TCP IPVS 监听器支持 full_nat、nat、dr。" location "body" type "String" optional true diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index b786ea8683b..ff0f7c34896 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -785,21 +785,21 @@ private boolean hasIpv6ServerIp(String serverGroupUuid) { private void validateTcpIpvsDoesNotUseIpv6Vip(LoadBalancerVO lbVO) { if (lbVO != null && !StringUtils.isEmpty(lbVO.getIpv6VipUuid())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 vip")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10180, "tcp ipvs listener doesn't support ipv6 vip")); } } private void validateTcpIpvsDoesNotUseIpv6ServerGroup(LoadBalancerServerGroupVO groupVO) { if (groupVO != null && groupVO.getIpVersion() != null && IPv6Constants.IPv6 == groupVO.getIpVersion()) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 server group")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10181, "tcp ipvs listener doesn't support ipv6 server group")); } } private void validateTcpIpvsDoesNotUseIpv6BackendIp(String ipAddress) { if (IPv6NetworkUtils.isIpv6Address(ipAddress)) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 backend server ip")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10182, "tcp ipvs listener doesn't support ipv6 backend server ip")); } } @@ -899,7 +899,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { if (LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane())) { if (!LB_PROTOCOL_TCP.equals(msg.getProtocol())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "data plane [%s] only supports tcp listener", msg.getDataPlane())); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10183, "data plane [%s] only supports tcp listener", msg.getDataPlane())); } validateTcpIpvsDoesNotUseIpv6Vip(lbVO); @@ -907,21 +907,25 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { msg.setForwardMode(LoadBalancerConstants.FORWARD_MODE_FULL_NAT); } - if (!LoadBalancerConstants.FORWARD_MODE_FULL_NAT.equals(msg.getForwardMode())) { + List supportedForwardModes = Arrays.asList( + LoadBalancerConstants.FORWARD_MODE_FULL_NAT, + LoadBalancerConstants.FORWARD_MODE_NAT, + LoadBalancerConstants.FORWARD_MODE_DR); + if (!supportedForwardModes.contains(msg.getForwardMode())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "TCP IPVS only supports forwardMode[%s] in current version, but got [%s]", - LoadBalancerConstants.FORWARD_MODE_FULL_NAT, msg.getForwardMode())); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10184, "TCP IPVS only supports forwardMode%s, but got [%s]", + supportedForwardModes, msg.getForwardMode())); } } else { if (!LoadBalancerConstants.DATA_PLANE_HAPROXY.equals(msg.getDataPlane())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "invalid dataPlane[%s], valid dataPlanes are [haproxy, ipvs]", msg.getDataPlane())); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10185, "invalid dataPlane[%s], valid dataPlanes are [haproxy, ipvs]", msg.getDataPlane())); } if (msg.getForwardMode() != null) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "forwardMode is only supported when dataPlane is ipvs")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10186, "forwardMode is only supported when dataPlane is ipvs")); } } String dataPlane = msg.getDataPlane(); @@ -960,7 +964,11 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && (hasHttpHealthCheckParameters(msg) || hasTag(msg, LoadBalancerSystemTags.HEALTH_PARAMETER))) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10187, "tcp ipvs listener doesn't support http health check parameters")); + } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.HEALTH_TIMEOUT)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10189, "tcp ipvs listener doesn't support healthCheckTimeout")); } if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) { @@ -1031,12 +1039,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { ) ); - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.HEALTH_TIMEOUT, - LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag( - map(e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.HEALTH_TIMEOUT.value(Long.class))) - ) - ); + if (!isTcpIpvsListener(msg.getProtocol(), dataPlane)) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.HEALTH_TIMEOUT, + LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag( + map(e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.HEALTH_TIMEOUT.value(Long.class))) + ) + ); + } insertTagIfNotExisting( msg, LoadBalancerSystemTags.UNHEALTHY_THRESHOLD, @@ -1451,6 +1461,11 @@ private void validate(APIRemoveCertificateFromLoadBalancerListenerMsg msg) { } private void validate(APIChangeLoadBalancerListenerMsg msg) { + if (msg.getForwardMode() != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10188, "forwardMode cannot be changed after load balancer listener is created")); + } + normalizeChangeHealthCheckTarget(msg); String target = msg.getHealthCheckTarget(); if (target != null) { @@ -1632,7 +1647,11 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { if (isTcpIpvsListener(listenerVO.getProtocol(), dataPlane) && hasHttpHealthCheckParameters(msg)) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10187, "tcp ipvs listener doesn't support http health check parameters")); + } + if (isTcpIpvsListener(listenerVO.getProtocol(), dataPlane) && msg.getHealthCheckTimeout() != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10189, "tcp ipvs listener doesn't support healthCheckTimeout")); } if (msg.getHealthCheckProtocol() != null) { @@ -2056,7 +2075,7 @@ private void validate(APIAddServerGroupToLoadBalancerListenerMsg msg){ validateTcpIpvsDoesNotUseIpv6ServerGroup(groupVO); if (hasIpv6ServerIp(msg.getServerGroupUuid())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 backend server ip")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10182, "tcp ipvs listener doesn't support ipv6 backend server ip")); } } if (listenerVO.getProtocol().equals(LB_PROTOCOL_UDP)) { diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java index 75a1e7c5ffc..0e78b167c0f 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java @@ -1862,7 +1862,9 @@ private void createListener(final APICreateLoadBalancerListenerMsg msg, final No tagMgr.createNonInherentSystemTags(msg.getSystemTags(), vo.getUuid(), LoadBalancerListenerVO.class.getSimpleName()); vo = dbf.updateAndRefresh(vo); - evt.setInventory(LoadBalancerListenerInventory.valueOf(vo)); + LoadBalancerListenerInventory inv = LoadBalancerListenerInventory.valueOf(vo); + inv.applyTcpIpvsSupportedParameterSystemTags(msg.getSystemTags()); + evt.setInventory(inv); bus.publish(evt); completion.done(); } @@ -2509,7 +2511,9 @@ public void run(MessageReply reply) { } } } else { - evt.setInventory(LoadBalancerListenerInventory.valueOf(lblVo)); + LoadBalancerListenerInventory inv = LoadBalancerListenerInventory.valueOf(lblVo); + inv.applyTcpIpvsSupportedParameters(msg.getBalancerAlgorithm(), msg.getMaxConnection()); + evt.setInventory(inv); } bus.publish(evt); } @@ -2518,7 +2522,9 @@ public void run(MessageReply reply) { chain.next(); return; } - evt.setInventory( LoadBalancerListenerInventory.valueOf(lblVo)); + LoadBalancerListenerInventory inv = LoadBalancerListenerInventory.valueOf(lblVo); + inv.applyTcpIpvsSupportedParameters(msg.getBalancerAlgorithm(), msg.getMaxConnection()); + evt.setInventory(inv); bus.publish(evt); chain.next(); } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java index 2111af073ea..c27457f4fc0 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java @@ -40,6 +40,8 @@ public class LoadBalancerListenerInventory implements Serializable { private String protocol; private String dataPlane; private String forwardMode; + private String balancerAlgorithm; + private Integer maxConnection; private String serverGroupUuid; private Timestamp createDate; private Timestamp lastOpDate; @@ -61,6 +63,13 @@ public static LoadBalancerListenerInventory valueOf(LoadBalancerListenerVO vo) { LoadBalancerConstants.DATA_PLANE_IPVS.equals(vo.getDataPlane())) { inv.setDataPlane(vo.getDataPlane()); inv.setForwardMode(vo.getForwardMode()); + inv.setBalancerAlgorithm(LoadBalancerSystemTags.BALANCER_ALGORITHM.getTokenByResourceUuid( + vo.getUuid(), LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN)); + String maxConnection = LoadBalancerSystemTags.MAX_CONNECTION.getTokenByResourceUuid( + vo.getUuid(), LoadBalancerSystemTags.MAX_CONNECTION_TOKEN); + if (maxConnection != null) { + inv.setMaxConnection(Integer.valueOf(maxConnection)); + } } inv.setSecurityPolicyType(vo.getSecurityPolicyType()); inv.setName(vo.getName()); @@ -99,6 +108,35 @@ public static List valueOf(Collection systemTags) { + if (systemTags == null || !LoadBalancerConstants.LB_PROTOCOL_TCP.equals(protocol) || + !LoadBalancerConstants.DATA_PLANE_IPVS.equals(dataPlane)) { + return; + } + + for (String tag : systemTags) { + if (tag.startsWith(LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN + "::")) { + setBalancerAlgorithm(tag.substring((LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN + "::").length())); + } else if (tag.startsWith(LoadBalancerSystemTags.MAX_CONNECTION_TOKEN + "::")) { + setMaxConnection(Integer.valueOf(tag.substring((LoadBalancerSystemTags.MAX_CONNECTION_TOKEN + "::").length()))); + } + } + } + public List getVmNicRefs() { return vmNicRefs; } @@ -187,6 +225,22 @@ public void setForwardMode(String forwardMode) { this.forwardMode = forwardMode; } + public String getBalancerAlgorithm() { + return balancerAlgorithm; + } + + public void setBalancerAlgorithm(String balancerAlgorithm) { + this.balancerAlgorithm = balancerAlgorithm; + } + + public Integer getMaxConnection() { + return maxConnection; + } + + public void setMaxConnection(Integer maxConnection) { + this.maxConnection = maxConnection; + } + public String getSecurityPolicyType() { return securityPolicyType; } diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java b/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java index 6ab8413099c..3186535a7b6 100644 --- a/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java +++ b/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java @@ -103,6 +103,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List httpCompressAlgos; + @Param(required = false, validValues = {"full_nat","nat","dr"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String forwardMode; + @Param(required = false) public java.util.List systemTags; diff --git a/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java b/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java index 5dc0eeecaec..a50b30f692e 100644 --- a/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java @@ -84,6 +84,22 @@ public java.lang.String getForwardMode() { return this.forwardMode; } + public java.lang.String balancerAlgorithm; + public void setBalancerAlgorithm(java.lang.String balancerAlgorithm) { + this.balancerAlgorithm = balancerAlgorithm; + } + public java.lang.String getBalancerAlgorithm() { + return this.balancerAlgorithm; + } + + public java.lang.Integer maxConnection; + public void setMaxConnection(java.lang.Integer maxConnection) { + this.maxConnection = maxConnection; + } + public java.lang.Integer getMaxConnection() { + return this.maxConnection; + } + public java.lang.String serverGroupUuid; public void setServerGroupUuid(java.lang.String serverGroupUuid) { this.serverGroupUuid = serverGroupUuid; diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy index cb55b118677..70631a2af93 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -30,6 +30,7 @@ import org.zstack.sdk.L3NetworkInventory import org.zstack.sdk.LoadBalancerInventory import org.zstack.sdk.LoadBalancerListenerInventory import org.zstack.sdk.LoadBalancerServerGroupInventory +import org.zstack.sdk.QueryLoadBalancerListenerAction import org.zstack.sdk.VipInventory import org.zstack.sdk.VmInstanceInventory import org.zstack.sdk.ZSClient @@ -205,7 +206,10 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { testTcpIpvsIpv6Validation() testTcpHaproxyDefaultDataPlane() testTcpIpvsFullNatListener() + testTcpIpvsSupportedParameterInventory() + testTcpIpvsNatAndDrListener() testTcpIpvsDefaultForwardMode() + testTcpIpvsForwardModeCannotBeChanged() testTcpIpvsCreateValidation() testTcpIpvsHealthCheckParameterValidation() testTcpHaproxyBackendRefreshPayload() @@ -250,6 +254,8 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { String rawResult = getApiResultString(result) assert !rawResult.contains("\"dataPlane\"") assert !rawResult.contains("\"forwardMode\"") + assert !rawResult.contains("\"balancerAlgorithm\"") + assert !rawResult.contains("\"maxConnection\"") LoadBalancerListenerInventory listener = result.getResult(CreateLoadBalancerListenerResult.class).inventory @@ -285,6 +291,108 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert vo.instancePort == 8080 } + void testTcpIpvsSupportedParameterInventory() { + CreateLoadBalancerListenerAction createAction = new CreateLoadBalancerListenerAction() + createAction.name = "tcp-ipvs-supported-parameters" + createAction.loadBalancerUuid = lb.uuid + createAction.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + createAction.loadBalancerPort = 11074 + createAction.instancePort = 8080 + createAction.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + createAction.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + createAction.systemTags = [ + "balancerAlgorithm::${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}".toString(), + "maxConnection::1234" + ] + createAction.sessionId = adminSession() + + ApiResult createResult = ZSClient.call(createAction) + assert createResult.error == null + String rawResult = getApiResultString(createResult) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}\"".toString()) + assert rawResult.contains("\"maxConnection\":1234") + + LoadBalancerListenerInventory listener = createResult.getResult(CreateLoadBalancerListenerResult.class).inventory + + assert listener.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert listener.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + + rawResult = queryListenerRaw(listener.uuid) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}\"".toString()) + assert rawResult.contains("\"maxConnection\":1234") + + String backendIp = backendIp("backend-vm-1") + LoadBalancerServerGroupInventory group = createServerGroupWithVmNics( + "tcp-ipvs-supported-parameters-group", + [backendVmNic("backend-vm-1", "100")]) + + int refreshOffset = refreshCmds.size() + addServerGroupToLoadBalancerListener { + listenerUuid = listener.uuid + serverGroupUuid = group.uuid + } + + VirtualRouterLoadBalancerBackend.LbTO to = lastLbTOWithParameters(listener.uuid, [ + "balancerAlgorithm::${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}".toString(), + "maxConnection::1234" + ], refreshOffset) + assertTcpIpvsTO(to, listener.uuid, 11074, LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN) + assert to.parameters.contains("maxConnection::1234") + assertServerGroups(to, [ + (group.uuid): [(backendIp): 100L] + ]) + + ChangeLoadBalancerListenerAction changeAction = new ChangeLoadBalancerListenerAction() + changeAction.uuid = listener.uuid + changeAction.balancerAlgorithm = LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN + changeAction.maxConnection = 2345 + changeAction.sessionId = adminSession() + + refreshOffset = refreshCmds.size() + ApiResult changeResult = ZSClient.call(changeAction) + assert changeResult.error == null + rawResult = getApiResultString(changeResult) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN}\"".toString()) + assert rawResult.contains("\"maxConnection\":2345") + + to = lastLbTOWithParameters(listener.uuid, [ + "balancerAlgorithm::${LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN}".toString(), + "maxConnection::2345" + ], refreshOffset) + assertTcpIpvsTO(to, listener.uuid, 11074, LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN) + assert to.parameters.contains("maxConnection::2345") + + rawResult = queryListenerRaw(listener.uuid) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN}\"".toString()) + assert rawResult.contains("\"maxConnection\":2345") + } + + void testTcpIpvsNatAndDrListener() { + [ + [11076, LoadBalancerConstants.FORWARD_MODE_NAT], + [11077, LoadBalancerConstants.FORWARD_MODE_DR] + ].each { List param -> + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-ipvs-${param[1]}" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = param[0] as int + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = param[1] as String + } + + assert listener.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert listener.forwardMode == param[1] + + LoadBalancerListenerVO vo = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listener.uuid) + .find() + assert vo.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert vo.forwardMode == param[1] + } + } + void testTcpIpvsIpv6Validation() { L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory addIpv6Range { @@ -312,6 +420,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { addIpv6Group.sessionId = adminSession() AddServerGroupToLoadBalancerListenerAction.Result addIpv6GroupResult = addIpv6Group.call() assert addIpv6GroupResult.error != null + assert addIpv6GroupResult.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10181" assert addIpv6GroupResult.error.details.contains("tcp ipvs listener doesn't support ipv6 server group") LoadBalancerServerGroupInventory ipv6GroupAfterListener = createLoadBalancerServerGroup { @@ -325,6 +434,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { addIpv6GroupAfterListener.sessionId = adminSession() AddServerGroupToLoadBalancerListenerAction.Result addIpv6GroupAfterListenerResult = addIpv6GroupAfterListener.call() assert addIpv6GroupAfterListenerResult.error != null + assert addIpv6GroupAfterListenerResult.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10181" assert addIpv6GroupAfterListenerResult.error.details.contains("tcp ipvs listener doesn't support ipv6 server group") LoadBalancerServerGroupInventory ipv4Group = createLoadBalancerServerGroup { @@ -341,6 +451,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { addIpv6BackendIp.sessionId = adminSession() AddBackendServerToServerGroupAction.Result addIpv6BackendIpResult = addIpv6BackendIp.call() assert addIpv6BackendIpResult.error != null + assert addIpv6BackendIpResult.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10182" assert addIpv6BackendIpResult.error.details.contains("tcp ipvs listener doesn't support ipv6 backend server ip") VipInventory ipv4Vip = createVip { @@ -371,6 +482,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { action.sessionId = adminSession() CreateLoadBalancerListenerAction.Result result = action.call() assert result.error != null + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10180" assert result.error.details.contains("tcp ipvs listener doesn't support ipv6 vip") } @@ -388,6 +500,29 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert listener.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT } + void testTcpIpvsForwardModeCannotBeChanged() { + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-ipvs-forward-mode-immutable" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11086 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + } + + ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.forwardMode = LoadBalancerConstants.FORWARD_MODE_NAT + } + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10188" + assert result.error.details.contains("forwardMode cannot be changed after load balancer listener is created") + + LoadBalancerListenerVO vo = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listener.uuid) + .find() + assert vo.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + } + void testTcpIpvsCreateValidation() { [ [11083, LoadBalancerConstants.LB_PROTOCOL_HTTP], @@ -396,18 +531,14 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { ].each { List param -> CreateLoadBalancerListenerAction.Result result = assertCreateListenerError(param[0] as int, param[1] as String, LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10183" assert result.error.details.contains("data plane [ipvs] only supports tcp listener") } - assertCreateListenerError(11085, LoadBalancerConstants.LB_PROTOCOL_TCP, + CreateLoadBalancerListenerAction.Result result = assertCreateListenerError(11085, LoadBalancerConstants.LB_PROTOCOL_TCP, LoadBalancerConstants.DATA_PLANE_HAPROXY, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) - CreateLoadBalancerListenerAction.Result natResult = assertCreateListenerError(11086, LoadBalancerConstants.LB_PROTOCOL_TCP, - LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_NAT) - assert natResult.error.details.contains("TCP IPVS only supports forwardMode[full_nat]") - - CreateLoadBalancerListenerAction.Result drResult = assertCreateListenerError(11087, LoadBalancerConstants.LB_PROTOCOL_TCP, - LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_DR) - assert drResult.error.details.contains("TCP IPVS only supports forwardMode[full_nat]") + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10186" + assert result.error.details.contains("forwardMode is only supported when dataPlane is ipvs") } void testTcpIpvsHealthCheckParameterValidation() { @@ -419,6 +550,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { { CreateLoadBalancerListenerAction action -> action.systemTags = ["healthCheckParameter::GET:/health:http_2xx"] } ].eachWithIndex { Closure setter, int index -> CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(11100 + index, setter) + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10187" assert result.error.details.contains("tcp ipvs listener doesn't support http health check parameters") } @@ -427,6 +559,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { } assertUnsupportedHealthCheckError(udpHealthCheckResult.error) + CreateLoadBalancerListenerAction.Result timeoutResult = assertCreateTcpIpvsListenerError(11106) { CreateLoadBalancerListenerAction action -> + action.systemTags = ["healthCheckTimeout::1"] + } + assertUnsupportedHealthCheckTimeoutError(timeoutResult.error) + LoadBalancerListenerInventory listener = createLoadBalancerListener { delegate.name = "tcp-ipvs-health-check-parameters" delegate.loadBalancerUuid = lb.uuid @@ -438,7 +575,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { delegate.systemTags = [ "healthCheckTarget::tcp:default", "healthCheckInterval::2", - "healthCheckTimeout::1", "healthyThreshold::2", "unhealthyThreshold::2" ] @@ -455,11 +591,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { VirtualRouterLoadBalancerBackend.LbTO to = lastLbTOWithParameters(listener.uuid, healthCheckParameters([ healthCheckTarget: "tcp:default", healthCheckInterval: "2", - healthCheckTimeout: "1", healthyThreshold: "2", unhealthyThreshold: "2" ]), createOffset) assertTcpIpvsTO(to, listener.uuid, 11110, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + assertNoHealthCheckTimeout(to) int changeOffset = refreshCmds.size() assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> @@ -470,7 +606,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "2", - healthCheckTimeout: "1", healthyThreshold: "2", unhealthyThreshold: "2" ]) @@ -484,24 +619,14 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "3", - healthCheckTimeout: "1", healthyThreshold: "2", unhealthyThreshold: "2" ]) - changeOffset = refreshCmds.size() - assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + ChangeLoadBalancerListenerAction.Result timeoutChangeResult = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> action.healthCheckTimeout = 2 } - assert LoadBalancerSystemTags.HEALTH_TIMEOUT.getTokenByResourceUuid(listener.uuid, - LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN) == "2" - assertHealthCheckPayload(listener.uuid, changeOffset, [ - healthCheckTarget: "tcp:8080", - healthCheckInterval: "3", - healthCheckTimeout: "2", - healthyThreshold: "2", - unhealthyThreshold: "2" - ]) + assertUnsupportedHealthCheckTimeoutError(timeoutChangeResult.error) changeOffset = refreshCmds.size() assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> @@ -512,7 +637,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "3", - healthCheckTimeout: "2", healthyThreshold: "3", unhealthyThreshold: "2" ]) @@ -526,7 +650,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "3", - healthCheckTimeout: "2", healthyThreshold: "3", unhealthyThreshold: "3" ]) @@ -538,6 +661,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { { ChangeLoadBalancerListenerAction action -> action.healthCheckProtocol = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_HTTP; action.healthCheckURI = "/health" } ].each { Closure setter -> ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid, setter) + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10187" assert result.error.details.contains("tcp ipvs listener doesn't support http health check parameters") } @@ -659,7 +783,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { ]) assert !to.enableFullLog - testTcpIpvsFullLogPayload(listener) + testTcpIpvsFullLogPayload(listener, group1) addBackendServerToServerGroup { serverGroupUuid = group1.uuid @@ -781,7 +905,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { } } - void testTcpIpvsFullLogPayload(LoadBalancerListenerInventory listener) { + void testTcpIpvsFullLogPayload(LoadBalancerListenerInventory listener, LoadBalancerServerGroupInventory group) { updateResourceConfig { category = VyosGlobalConfig.CATEGORY name = VyosGlobalConfig.ENABLE_LOADBALANCER_FULL_LOG.name @@ -795,6 +919,16 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertTcpIpvsTO(to, listener.uuid, 11090, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) assert to.enableFullLog + LoadBalancerListenerInventory disabledFullLogListener = createTcpIpvsListener( + "tcp-ipvs-full-log-disabled", 11093, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + addServerGroupToLoadBalancerListener { + listenerUuid = disabledFullLogListener.uuid + serverGroupUuid = group.uuid + } + to = lastLbTO(disabledFullLogListener.uuid) + assertTcpIpvsTO(to, disabledFullLogListener.uuid, 11093, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + assert !to.enableFullLog + updateResourceConfig { category = VyosGlobalConfig.CATEGORY name = VyosGlobalConfig.ENABLE_LOADBALANCER_FULL_LOG.name @@ -814,18 +948,22 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { healthCheckParameters(expected).each { String parameter -> assert to.parameters.contains(parameter) } + assertNoHealthCheckTimeout(to) } List healthCheckParameters(Map values) { return [ "healthCheckTarget::${values.healthCheckTarget}".toString(), "healthCheckInterval::${values.healthCheckInterval}".toString(), - "healthCheckTimeout::${values.healthCheckTimeout}".toString(), "healthyThreshold::${values.healthyThreshold}".toString(), "unhealthyThreshold::${values.unhealthyThreshold}".toString() ] } + void assertNoHealthCheckTimeout(VirtualRouterLoadBalancerBackend.LbTO to) { + assert !to.parameters.any { String parameter -> parameter.startsWith("healthCheckTimeout::") } + } + LoadBalancerListenerInventory createTcpIpvsListener(String name, int port, String algorithm) { return createLoadBalancerListener { delegate.name = name @@ -1010,12 +1148,27 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { error.details.contains("不支持此类型") } + void assertUnsupportedHealthCheckTimeoutError(def error) { + assert error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10189" + assert error.details.contains("tcp ipvs listener doesn't support healthCheckTimeout") + } + String getApiResultString(ApiResult result) { def field = ApiResult.class.getDeclaredField("resultString") field.setAccessible(true) return field.get(result) as String } + String queryListenerRaw(String listenerUuid) { + QueryLoadBalancerListenerAction action = new QueryLoadBalancerListenerAction() + action.conditions = ["uuid=${listenerUuid}".toString()] + action.sessionId = adminSession() + + ApiResult result = ZSClient.call(action) + assert result.error == null + return getApiResultString(result) + } + @Override void clean() { env.delete() diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 24d0b031e3c..7d25a93b28a 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -13893,6 +13893,26 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10179 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10179"; + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10180 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10180"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10181 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10181"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10182 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10182"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10183 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10183"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10184 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10184"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10185 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10185"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10186 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10186"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10187 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10187"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10188 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10188"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10189 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10189"; + public static final String ORG_ZSTACK_IPSEC_10000 = "ORG_ZSTACK_IPSEC_10000"; public static final String ORG_ZSTACK_IPSEC_10001 = "ORG_ZSTACK_IPSEC_10001";