diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java index 6e47b12bebe..7a4ae4622a2 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java @@ -95,6 +95,9 @@ public class APIChangeLoadBalancerListenerMsg extends APIMessage implements Load @APIParam(required = false) private List httpCompressAlgos; + @APIParam(validValues = {LoadBalancerConstants.FORWARD_MODE_FULL_NAT, LoadBalancerConstants.FORWARD_MODE_NAT, LoadBalancerConstants.FORWARD_MODE_DR}, required = false) + private String forwardMode; + @APINoSee private String loadBalancerUuid; @@ -318,6 +321,14 @@ public void setHttpCompressAlgos(List httpCompressAlgos) { this.httpCompressAlgos = httpCompressAlgos; } + public String getForwardMode() { + return forwardMode; + } + + public void setForwardMode(String forwardMode) { + this.forwardMode = forwardMode; + } + public static APIChangeLoadBalancerListenerMsg __example__() { APIChangeLoadBalancerListenerMsg msg = new APIChangeLoadBalancerListenerMsg(); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy index bf2c1f80054..f218b996f5c 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy @@ -141,6 +141,16 @@ doc { optional true since "3.9" } + column { + name "forwardMode" + enclosedIn "changeLoadBalancerListener" + desc "监听器使用 IPVS 数据平面时的转发模式。该参数只允许在创建监听器时指定，创建后不允许修改。" + location "body" + type "String" + optional true + since "5.5.28" + values ("full_nat","nat","dr") + } column { name "aclStatus" enclosedIn "changeLoadBalancerListener" diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy index 099aa923130..8e618f46cb6 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy @@ -238,7 +238,7 @@ doc { column { name "forwardMode" enclosedIn "params" - desc "监听器使用 IPVS 数据平面时的转发模式。当前 TCP IPVS 监听器支持 full_nat。" + desc "监听器使用 IPVS 数据平面时的转发模式。当前 TCP IPVS 监听器支持 full_nat、nat、dr。" location "body" type "String" optional true diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index b786ea8683b..ff0f7c34896 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -785,21 +785,21 @@ private boolean hasIpv6ServerIp(String serverGroupUuid) { private void validateTcpIpvsDoesNotUseIpv6Vip(LoadBalancerVO lbVO) { if (lbVO != null && !StringUtils.isEmpty(lbVO.getIpv6VipUuid())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 vip")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10180, "tcp ipvs listener doesn't support ipv6 vip")); } } private void validateTcpIpvsDoesNotUseIpv6ServerGroup(LoadBalancerServerGroupVO groupVO) { if (groupVO != null && groupVO.getIpVersion() != null && IPv6Constants.IPv6 == groupVO.getIpVersion()) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 server group")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10181, "tcp ipvs listener doesn't support ipv6 server group")); } } private void validateTcpIpvsDoesNotUseIpv6BackendIp(String ipAddress) { if (IPv6NetworkUtils.isIpv6Address(ipAddress)) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 backend server ip")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10182, "tcp ipvs listener doesn't support ipv6 backend server ip")); } } @@ -899,7 +899,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { if (LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane())) { if (!LB_PROTOCOL_TCP.equals(msg.getProtocol())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "data plane [%s] only supports tcp listener", msg.getDataPlane())); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10183, "data plane [%s] only supports tcp listener", msg.getDataPlane())); } validateTcpIpvsDoesNotUseIpv6Vip(lbVO); @@ -907,21 +907,25 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { msg.setForwardMode(LoadBalancerConstants.FORWARD_MODE_FULL_NAT); } - if (!LoadBalancerConstants.FORWARD_MODE_FULL_NAT.equals(msg.getForwardMode())) { + List supportedForwardModes = Arrays.asList( + LoadBalancerConstants.FORWARD_MODE_FULL_NAT, + LoadBalancerConstants.FORWARD_MODE_NAT, + LoadBalancerConstants.FORWARD_MODE_DR); + if (!supportedForwardModes.contains(msg.getForwardMode())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "TCP IPVS only supports forwardMode[%s] in current version, but got [%s]", - LoadBalancerConstants.FORWARD_MODE_FULL_NAT, msg.getForwardMode())); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10184, "TCP IPVS only supports forwardMode%s, but got [%s]", + supportedForwardModes, msg.getForwardMode())); } } else { if (!LoadBalancerConstants.DATA_PLANE_HAPROXY.equals(msg.getDataPlane())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "invalid dataPlane[%s], valid dataPlanes are [haproxy, ipvs]", msg.getDataPlane())); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10185, "invalid dataPlane[%s], valid dataPlanes are [haproxy, ipvs]", msg.getDataPlane())); } if (msg.getForwardMode() != null) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "forwardMode is only supported when dataPlane is ipvs")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10186, "forwardMode is only supported when dataPlane is ipvs")); } } String dataPlane = msg.getDataPlane(); @@ -960,7 +964,11 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && (hasHttpHealthCheckParameters(msg) || hasTag(msg, LoadBalancerSystemTags.HEALTH_PARAMETER))) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10187, "tcp ipvs listener doesn't support http health check parameters")); + } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.HEALTH_TIMEOUT)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10189, "tcp ipvs listener doesn't support healthCheckTimeout")); } if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) { @@ -1031,12 +1039,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { ) ); - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.HEALTH_TIMEOUT, - LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag( - map(e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.HEALTH_TIMEOUT.value(Long.class))) - ) - ); + if (!isTcpIpvsListener(msg.getProtocol(), dataPlane)) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.HEALTH_TIMEOUT, + LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag( + map(e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.HEALTH_TIMEOUT.value(Long.class))) + ) + ); + } insertTagIfNotExisting( msg, LoadBalancerSystemTags.UNHEALTHY_THRESHOLD, @@ -1451,6 +1461,11 @@ private void validate(APIRemoveCertificateFromLoadBalancerListenerMsg msg) { } private void validate(APIChangeLoadBalancerListenerMsg msg) { + if (msg.getForwardMode() != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10188, "forwardMode cannot be changed after load balancer listener is created")); + } + normalizeChangeHealthCheckTarget(msg); String target = msg.getHealthCheckTarget(); if (target != null) { @@ -1632,7 +1647,11 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { if (isTcpIpvsListener(listenerVO.getProtocol(), dataPlane) && hasHttpHealthCheckParameters(msg)) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10187, "tcp ipvs listener doesn't support http health check parameters")); + } + if (isTcpIpvsListener(listenerVO.getProtocol(), dataPlane) && msg.getHealthCheckTimeout() != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10189, "tcp ipvs listener doesn't support healthCheckTimeout")); } if (msg.getHealthCheckProtocol() != null) { @@ -2056,7 +2075,7 @@ private void validate(APIAddServerGroupToLoadBalancerListenerMsg msg){ validateTcpIpvsDoesNotUseIpv6ServerGroup(groupVO); if (hasIpv6ServerIp(msg.getServerGroupUuid())) { throw new ApiMessageInterceptionException( - operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support ipv6 backend server ip")); + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10182, "tcp ipvs listener doesn't support ipv6 backend server ip")); } } if (listenerVO.getProtocol().equals(LB_PROTOCOL_UDP)) { diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java index 75a1e7c5ffc..0e78b167c0f 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java @@ -1862,7 +1862,9 @@ private void createListener(final APICreateLoadBalancerListenerMsg msg, final No tagMgr.createNonInherentSystemTags(msg.getSystemTags(), vo.getUuid(), LoadBalancerListenerVO.class.getSimpleName()); vo = dbf.updateAndRefresh(vo); - evt.setInventory(LoadBalancerListenerInventory.valueOf(vo)); + LoadBalancerListenerInventory inv = LoadBalancerListenerInventory.valueOf(vo); + inv.applyTcpIpvsSupportedParameterSystemTags(msg.getSystemTags()); + evt.setInventory(inv); bus.publish(evt); completion.done(); } @@ -2509,7 +2511,9 @@ public void run(MessageReply reply) { } } } else { - evt.setInventory(LoadBalancerListenerInventory.valueOf(lblVo)); + LoadBalancerListenerInventory inv = LoadBalancerListenerInventory.valueOf(lblVo); + inv.applyTcpIpvsSupportedParameters(msg.getBalancerAlgorithm(), msg.getMaxConnection()); + evt.setInventory(inv); } bus.publish(evt); } @@ -2518,7 +2522,9 @@ public void run(MessageReply reply) { chain.next(); return; } - evt.setInventory( LoadBalancerListenerInventory.valueOf(lblVo)); + LoadBalancerListenerInventory inv = LoadBalancerListenerInventory.valueOf(lblVo); + inv.applyTcpIpvsSupportedParameters(msg.getBalancerAlgorithm(), msg.getMaxConnection()); + evt.setInventory(inv); bus.publish(evt); chain.next(); } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java index 2111af073ea..c27457f4fc0 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java @@ -40,6 +40,8 @@ public class LoadBalancerListenerInventory implements Serializable { private String protocol; private String dataPlane; private String forwardMode; + private String balancerAlgorithm; + private Integer maxConnection; private String serverGroupUuid; private Timestamp createDate; private Timestamp lastOpDate; @@ -61,6 +63,13 @@ public static LoadBalancerListenerInventory valueOf(LoadBalancerListenerVO vo) { LoadBalancerConstants.DATA_PLANE_IPVS.equals(vo.getDataPlane())) { inv.setDataPlane(vo.getDataPlane()); inv.setForwardMode(vo.getForwardMode()); + inv.setBalancerAlgorithm(LoadBalancerSystemTags.BALANCER_ALGORITHM.getTokenByResourceUuid( + vo.getUuid(), LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN)); + String maxConnection = LoadBalancerSystemTags.MAX_CONNECTION.getTokenByResourceUuid( + vo.getUuid(), LoadBalancerSystemTags.MAX_CONNECTION_TOKEN); + if (maxConnection != null) { + inv.setMaxConnection(Integer.valueOf(maxConnection)); + } } inv.setSecurityPolicyType(vo.getSecurityPolicyType()); inv.setName(vo.getName()); @@ -99,6 +108,35 @@ public static List valueOf(Collection systemTags) { + if (systemTags == null || !LoadBalancerConstants.LB_PROTOCOL_TCP.equals(protocol) || + !LoadBalancerConstants.DATA_PLANE_IPVS.equals(dataPlane)) { + return; + } + + for (String tag : systemTags) { + if (tag.startsWith(LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN + "::")) { + setBalancerAlgorithm(tag.substring((LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN + "::").length())); + } else if (tag.startsWith(LoadBalancerSystemTags.MAX_CONNECTION_TOKEN + "::")) { + setMaxConnection(Integer.valueOf(tag.substring((LoadBalancerSystemTags.MAX_CONNECTION_TOKEN + "::").length()))); + } + } + } + public List getVmNicRefs() { return vmNicRefs; } @@ -187,6 +225,22 @@ public void setForwardMode(String forwardMode) { this.forwardMode = forwardMode; } + public String getBalancerAlgorithm() { + return balancerAlgorithm; + } + + public void setBalancerAlgorithm(String balancerAlgorithm) { + this.balancerAlgorithm = balancerAlgorithm; + } + + public Integer getMaxConnection() { + return maxConnection; + } + + public void setMaxConnection(Integer maxConnection) { + this.maxConnection = maxConnection; + } + public String getSecurityPolicyType() { return securityPolicyType; } diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java b/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java index 6ab8413099c..3186535a7b6 100644 --- a/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java +++ b/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java @@ -103,6 +103,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List httpCompressAlgos; + @Param(required = false, validValues = {"full_nat","nat","dr"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String forwardMode; + @Param(required = false) public java.util.List systemTags; diff --git a/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java b/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java index 5dc0eeecaec..a50b30f692e 100644 --- a/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java @@ -84,6 +84,22 @@ public java.lang.String getForwardMode() { return this.forwardMode; } + public java.lang.String balancerAlgorithm; + public void setBalancerAlgorithm(java.lang.String balancerAlgorithm) { + this.balancerAlgorithm = balancerAlgorithm; + } + public java.lang.String getBalancerAlgorithm() { + return this.balancerAlgorithm; + } + + public java.lang.Integer maxConnection; + public void setMaxConnection(java.lang.Integer maxConnection) { + this.maxConnection = maxConnection; + } + public java.lang.Integer getMaxConnection() { + return this.maxConnection; + } + public java.lang.String serverGroupUuid; public void setServerGroupUuid(java.lang.String serverGroupUuid) { this.serverGroupUuid = serverGroupUuid; diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy index cb55b118677..70631a2af93 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -30,6 +30,7 @@ import org.zstack.sdk.L3NetworkInventory import org.zstack.sdk.LoadBalancerInventory import org.zstack.sdk.LoadBalancerListenerInventory import org.zstack.sdk.LoadBalancerServerGroupInventory +import org.zstack.sdk.QueryLoadBalancerListenerAction import org.zstack.sdk.VipInventory import org.zstack.sdk.VmInstanceInventory import org.zstack.sdk.ZSClient @@ -205,7 +206,10 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { testTcpIpvsIpv6Validation() testTcpHaproxyDefaultDataPlane() testTcpIpvsFullNatListener() + testTcpIpvsSupportedParameterInventory() + testTcpIpvsNatAndDrListener() testTcpIpvsDefaultForwardMode() + testTcpIpvsForwardModeCannotBeChanged() testTcpIpvsCreateValidation() testTcpIpvsHealthCheckParameterValidation() testTcpHaproxyBackendRefreshPayload() @@ -250,6 +254,8 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { String rawResult = getApiResultString(result) assert !rawResult.contains("\"dataPlane\"") assert !rawResult.contains("\"forwardMode\"") + assert !rawResult.contains("\"balancerAlgorithm\"") + assert !rawResult.contains("\"maxConnection\"") LoadBalancerListenerInventory listener = result.getResult(CreateLoadBalancerListenerResult.class).inventory @@ -285,6 +291,108 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert vo.instancePort == 8080 } + void testTcpIpvsSupportedParameterInventory() { + CreateLoadBalancerListenerAction createAction = new CreateLoadBalancerListenerAction() + createAction.name = "tcp-ipvs-supported-parameters" + createAction.loadBalancerUuid = lb.uuid + createAction.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + createAction.loadBalancerPort = 11074 + createAction.instancePort = 8080 + createAction.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + createAction.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + createAction.systemTags = [ + "balancerAlgorithm::${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}".toString(), + "maxConnection::1234" + ] + createAction.sessionId = adminSession() + + ApiResult createResult = ZSClient.call(createAction) + assert createResult.error == null + String rawResult = getApiResultString(createResult) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}\"".toString()) + assert rawResult.contains("\"maxConnection\":1234") + + LoadBalancerListenerInventory listener = createResult.getResult(CreateLoadBalancerListenerResult.class).inventory + + assert listener.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert listener.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + + rawResult = queryListenerRaw(listener.uuid) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}\"".toString()) + assert rawResult.contains("\"maxConnection\":1234") + + String backendIp = backendIp("backend-vm-1") + LoadBalancerServerGroupInventory group = createServerGroupWithVmNics( + "tcp-ipvs-supported-parameters-group", + [backendVmNic("backend-vm-1", "100")]) + + int refreshOffset = refreshCmds.size() + addServerGroupToLoadBalancerListener { + listenerUuid = listener.uuid + serverGroupUuid = group.uuid + } + + VirtualRouterLoadBalancerBackend.LbTO to = lastLbTOWithParameters(listener.uuid, [ + "balancerAlgorithm::${LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN}".toString(), + "maxConnection::1234" + ], refreshOffset) + assertTcpIpvsTO(to, listener.uuid, 11074, LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN) + assert to.parameters.contains("maxConnection::1234") + assertServerGroups(to, [ + (group.uuid): [(backendIp): 100L] + ]) + + ChangeLoadBalancerListenerAction changeAction = new ChangeLoadBalancerListenerAction() + changeAction.uuid = listener.uuid + changeAction.balancerAlgorithm = LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN + changeAction.maxConnection = 2345 + changeAction.sessionId = adminSession() + + refreshOffset = refreshCmds.size() + ApiResult changeResult = ZSClient.call(changeAction) + assert changeResult.error == null + rawResult = getApiResultString(changeResult) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN}\"".toString()) + assert rawResult.contains("\"maxConnection\":2345") + + to = lastLbTOWithParameters(listener.uuid, [ + "balancerAlgorithm::${LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN}".toString(), + "maxConnection::2345" + ], refreshOffset) + assertTcpIpvsTO(to, listener.uuid, 11074, LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN) + assert to.parameters.contains("maxConnection::2345") + + rawResult = queryListenerRaw(listener.uuid) + assert rawResult.contains("\"balancerAlgorithm\":\"${LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN}\"".toString()) + assert rawResult.contains("\"maxConnection\":2345") + } + + void testTcpIpvsNatAndDrListener() { + [ + [11076, LoadBalancerConstants.FORWARD_MODE_NAT], + [11077, LoadBalancerConstants.FORWARD_MODE_DR] + ].each { List param -> + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-ipvs-${param[1]}" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = param[0] as int + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = param[1] as String + } + + assert listener.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert listener.forwardMode == param[1] + + LoadBalancerListenerVO vo = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listener.uuid) + .find() + assert vo.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert vo.forwardMode == param[1] + } + } + void testTcpIpvsIpv6Validation() { L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory addIpv6Range { @@ -312,6 +420,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { addIpv6Group.sessionId = adminSession() AddServerGroupToLoadBalancerListenerAction.Result addIpv6GroupResult = addIpv6Group.call() assert addIpv6GroupResult.error != null + assert addIpv6GroupResult.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10181" assert addIpv6GroupResult.error.details.contains("tcp ipvs listener doesn't support ipv6 server group") LoadBalancerServerGroupInventory ipv6GroupAfterListener = createLoadBalancerServerGroup { @@ -325,6 +434,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { addIpv6GroupAfterListener.sessionId = adminSession() AddServerGroupToLoadBalancerListenerAction.Result addIpv6GroupAfterListenerResult = addIpv6GroupAfterListener.call() assert addIpv6GroupAfterListenerResult.error != null + assert addIpv6GroupAfterListenerResult.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10181" assert addIpv6GroupAfterListenerResult.error.details.contains("tcp ipvs listener doesn't support ipv6 server group") LoadBalancerServerGroupInventory ipv4Group = createLoadBalancerServerGroup { @@ -341,6 +451,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { addIpv6BackendIp.sessionId = adminSession() AddBackendServerToServerGroupAction.Result addIpv6BackendIpResult = addIpv6BackendIp.call() assert addIpv6BackendIpResult.error != null + assert addIpv6BackendIpResult.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10182" assert addIpv6BackendIpResult.error.details.contains("tcp ipvs listener doesn't support ipv6 backend server ip") VipInventory ipv4Vip = createVip { @@ -371,6 +482,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { action.sessionId = adminSession() CreateLoadBalancerListenerAction.Result result = action.call() assert result.error != null + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10180" assert result.error.details.contains("tcp ipvs listener doesn't support ipv6 vip") } @@ -388,6 +500,29 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert listener.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT } + void testTcpIpvsForwardModeCannotBeChanged() { + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-ipvs-forward-mode-immutable" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11086 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + } + + ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.forwardMode = LoadBalancerConstants.FORWARD_MODE_NAT + } + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10188" + assert result.error.details.contains("forwardMode cannot be changed after load balancer listener is created") + + LoadBalancerListenerVO vo = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listener.uuid) + .find() + assert vo.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + } + void testTcpIpvsCreateValidation() { [ [11083, LoadBalancerConstants.LB_PROTOCOL_HTTP], @@ -396,18 +531,14 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { ].each { List param -> CreateLoadBalancerListenerAction.Result result = assertCreateListenerError(param[0] as int, param[1] as String, LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10183" assert result.error.details.contains("data plane [ipvs] only supports tcp listener") } - assertCreateListenerError(11085, LoadBalancerConstants.LB_PROTOCOL_TCP, + CreateLoadBalancerListenerAction.Result result = assertCreateListenerError(11085, LoadBalancerConstants.LB_PROTOCOL_TCP, LoadBalancerConstants.DATA_PLANE_HAPROXY, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) - CreateLoadBalancerListenerAction.Result natResult = assertCreateListenerError(11086, LoadBalancerConstants.LB_PROTOCOL_TCP, - LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_NAT) - assert natResult.error.details.contains("TCP IPVS only supports forwardMode[full_nat]") - - CreateLoadBalancerListenerAction.Result drResult = assertCreateListenerError(11087, LoadBalancerConstants.LB_PROTOCOL_TCP, - LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_DR) - assert drResult.error.details.contains("TCP IPVS only supports forwardMode[full_nat]") + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10186" + assert result.error.details.contains("forwardMode is only supported when dataPlane is ipvs") } void testTcpIpvsHealthCheckParameterValidation() { @@ -419,6 +550,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { { CreateLoadBalancerListenerAction action -> action.systemTags = ["healthCheckParameter::GET:/health:http_2xx"] } ].eachWithIndex { Closure setter, int index -> CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(11100 + index, setter) + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10187" assert result.error.details.contains("tcp ipvs listener doesn't support http health check parameters") } @@ -427,6 +559,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { } assertUnsupportedHealthCheckError(udpHealthCheckResult.error) + CreateLoadBalancerListenerAction.Result timeoutResult = assertCreateTcpIpvsListenerError(11106) { CreateLoadBalancerListenerAction action -> + action.systemTags = ["healthCheckTimeout::1"] + } + assertUnsupportedHealthCheckTimeoutError(timeoutResult.error) + LoadBalancerListenerInventory listener = createLoadBalancerListener { delegate.name = "tcp-ipvs-health-check-parameters" delegate.loadBalancerUuid = lb.uuid @@ -438,7 +575,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { delegate.systemTags = [ "healthCheckTarget::tcp:default", "healthCheckInterval::2", - "healthCheckTimeout::1", "healthyThreshold::2", "unhealthyThreshold::2" ] @@ -455,11 +591,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { VirtualRouterLoadBalancerBackend.LbTO to = lastLbTOWithParameters(listener.uuid, healthCheckParameters([ healthCheckTarget: "tcp:default", healthCheckInterval: "2", - healthCheckTimeout: "1", healthyThreshold: "2", unhealthyThreshold: "2" ]), createOffset) assertTcpIpvsTO(to, listener.uuid, 11110, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + assertNoHealthCheckTimeout(to) int changeOffset = refreshCmds.size() assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> @@ -470,7 +606,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "2", - healthCheckTimeout: "1", healthyThreshold: "2", unhealthyThreshold: "2" ]) @@ -484,24 +619,14 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "3", - healthCheckTimeout: "1", healthyThreshold: "2", unhealthyThreshold: "2" ]) - changeOffset = refreshCmds.size() - assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + ChangeLoadBalancerListenerAction.Result timeoutChangeResult = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> action.healthCheckTimeout = 2 } - assert LoadBalancerSystemTags.HEALTH_TIMEOUT.getTokenByResourceUuid(listener.uuid, - LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN) == "2" - assertHealthCheckPayload(listener.uuid, changeOffset, [ - healthCheckTarget: "tcp:8080", - healthCheckInterval: "3", - healthCheckTimeout: "2", - healthyThreshold: "2", - unhealthyThreshold: "2" - ]) + assertUnsupportedHealthCheckTimeoutError(timeoutChangeResult.error) changeOffset = refreshCmds.size() assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> @@ -512,7 +637,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "3", - healthCheckTimeout: "2", healthyThreshold: "3", unhealthyThreshold: "2" ]) @@ -526,7 +650,6 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertHealthCheckPayload(listener.uuid, changeOffset, [ healthCheckTarget: "tcp:8080", healthCheckInterval: "3", - healthCheckTimeout: "2", healthyThreshold: "3", unhealthyThreshold: "3" ]) @@ -538,6 +661,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { { ChangeLoadBalancerListenerAction action -> action.healthCheckProtocol = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_HTTP; action.healthCheckURI = "/health" } ].each { Closure setter -> ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid, setter) + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10187" assert result.error.details.contains("tcp ipvs listener doesn't support http health check parameters") } @@ -659,7 +783,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { ]) assert !to.enableFullLog - testTcpIpvsFullLogPayload(listener) + testTcpIpvsFullLogPayload(listener, group1) addBackendServerToServerGroup { serverGroupUuid = group1.uuid @@ -781,7 +905,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { } } - void testTcpIpvsFullLogPayload(LoadBalancerListenerInventory listener) { + void testTcpIpvsFullLogPayload(LoadBalancerListenerInventory listener, LoadBalancerServerGroupInventory group) { updateResourceConfig { category = VyosGlobalConfig.CATEGORY name = VyosGlobalConfig.ENABLE_LOADBALANCER_FULL_LOG.name @@ -795,6 +919,16 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assertTcpIpvsTO(to, listener.uuid, 11090, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) assert to.enableFullLog + LoadBalancerListenerInventory disabledFullLogListener = createTcpIpvsListener( + "tcp-ipvs-full-log-disabled", 11093, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + addServerGroupToLoadBalancerListener { + listenerUuid = disabledFullLogListener.uuid + serverGroupUuid = group.uuid + } + to = lastLbTO(disabledFullLogListener.uuid) + assertTcpIpvsTO(to, disabledFullLogListener.uuid, 11093, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + assert !to.enableFullLog + updateResourceConfig { category = VyosGlobalConfig.CATEGORY name = VyosGlobalConfig.ENABLE_LOADBALANCER_FULL_LOG.name @@ -814,18 +948,22 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { healthCheckParameters(expected).each { String parameter -> assert to.parameters.contains(parameter) } + assertNoHealthCheckTimeout(to) } List healthCheckParameters(Map values) { return [ "healthCheckTarget::${values.healthCheckTarget}".toString(), "healthCheckInterval::${values.healthCheckInterval}".toString(), - "healthCheckTimeout::${values.healthCheckTimeout}".toString(), "healthyThreshold::${values.healthyThreshold}".toString(), "unhealthyThreshold::${values.unhealthyThreshold}".toString() ] } + void assertNoHealthCheckTimeout(VirtualRouterLoadBalancerBackend.LbTO to) { + assert !to.parameters.any { String parameter -> parameter.startsWith("healthCheckTimeout::") } + } + LoadBalancerListenerInventory createTcpIpvsListener(String name, int port, String algorithm) { return createLoadBalancerListener { delegate.name = name @@ -1010,12 +1148,27 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { error.details.contains("不支持此类型") } + void assertUnsupportedHealthCheckTimeoutError(def error) { + assert error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10189" + assert error.details.contains("tcp ipvs listener doesn't support healthCheckTimeout") + } + String getApiResultString(ApiResult result) { def field = ApiResult.class.getDeclaredField("resultString") field.setAccessible(true) return field.get(result) as String } + String queryListenerRaw(String listenerUuid) { + QueryLoadBalancerListenerAction action = new QueryLoadBalancerListenerAction() + action.conditions = ["uuid=${listenerUuid}".toString()] + action.sessionId = adminSession() + + ApiResult result = ZSClient.call(action) + assert result.error == null + return getApiResultString(result) + } + @Override void clean() { env.delete() diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 24d0b031e3c..7d25a93b28a 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -13893,6 +13893,26 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10179 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10179"; + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10180 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10180"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10181 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10181"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10182 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10182"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10183 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10183"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10184 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10184"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10185 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10185"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10186 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10186"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10187 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10187"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10188 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10188"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10189 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10189"; + public static final String ORG_ZSTACK_IPSEC_10000 = "ORG_ZSTACK_IPSEC_10000"; public static final String ORG_ZSTACK_IPSEC_10001 = "ORG_ZSTACK_IPSEC_10001";