Skip to content

<fix>[console]: ZSTAC-87566 delete expired DPU VNC proxy before recreation #4685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zstack-robot-2 wants to merge 1 commit into
base: feature-5.5.38-dpu2
Choose a base branch
from
Open

<fix>[console]: ZSTAC-87566 delete expired DPU VNC proxy before recreation #4685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 31 additions & 4 deletions console/src/main/java/org/zstack/console/AbstractConsoleProxyBackend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.zstack.header.Component;
import org.zstack.header.console.ConsoleBackend;
import org.zstack.header.console.ConsoleConstants;
import org.zstack.header.console.ConsoleHypervisorBackend;
import org.zstack.header.console.ConsoleInventory;
import org.zstack.header.console.ConsoleProxy;
import org.zstack.header.console.ConsoleProxyInventory;
Expand All @@ -38,6 +39,7 @@
import org.zstack.header.identity.SessionInventory;
import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint;
import org.zstack.header.rest.RESTFacade;
import org.zstack.header.host.HypervisorType;
import org.zstack.header.vm.VmInstanceInventory;
import org.zstack.header.vm.VmInstanceVO;
import org.zstack.utils.Utils;
Expand Down Expand Up @@ -72,6 +74,8 @@ public abstract class AbstractConsoleProxyBackend implements ConsoleBackend, Com
protected ErrorFacade errf;
@Autowired
protected Timer timer;
@Autowired
protected ConsoleManager consoleMgr;

protected static final String ANSIBLE_PLAYBOOK_NAME = "consoleproxy.py";

Expand Down Expand Up @@ -139,9 +143,26 @@ public void grantConsoleAccess(final SessionInventory session, final VmInstanceI
}

if (timer.getCurrentTimestamp().after(vo.getExpiredDate())) {
dbf.remove(vo);
ConsoleProxy proxy = getConsoleProxy(session, vm);
establishNewProxy(proxy, vm, complete);
ConsoleHypervisorBackend backend = consoleMgr.getHypervisorConsoleBackend(HypervisorType.valueOf(vm.getHypervisorType()));
if (!backend.requireExclusiveConsoleSessionRenewal()) {
dbf.remove(vo);
ConsoleProxy proxy = getConsoleProxy(session, vm);
establishNewProxy(proxy, vm, complete);
return;
}

deleteConsoleSession(vm, false, new Completion(complete) {
@Override
public void success() {
ConsoleProxy proxy = getConsoleProxy(session, vm);
establishNewProxy(proxy, vm, complete);
}

@Override
public void fail(ErrorCode errorCode) {
complete.fail(errorCode);
}
});
return;
}

Expand Down Expand Up @@ -287,6 +308,10 @@ public void fail(ErrorCode errorCode) {

@Override
public void deleteConsoleSession(final VmInstanceInventory vm, final Completion completion) {
deleteConsoleSession(vm, true, completion);
}

private void deleteConsoleSession(final VmInstanceInventory vm, boolean removeVoOnFailure, final Completion completion) {
SimpleQuery<ConsoleProxyVO> q = dbf.createQuery(ConsoleProxyVO.class);
q.add(ConsoleProxyVO_.vmInstanceUuid, SimpleQuery.Op.EQ, vm.getUuid());
q.add(ConsoleProxyVO_.status, SimpleQuery.Op.EQ, ConsoleProxyStatus.Active);
Expand Down Expand Up @@ -317,7 +342,9 @@ public void fail(ErrorCode errorCode) {
gc.NAME = String.format("delete-console-proxy-%s", vo.getUuid());
gc.consoleProxy = ConsoleProxyInventory.valueOf(vo);
gc.submit(ConsoleGlobalConfig.DELETE_CONSOLE_PROXY_RETRY_DELAY.value(Long.class), TimeUnit.SECONDS);
dbf.remove(vo);
if (removeVoOnFailure) {
dbf.remove(vo);
}
completion.fail(errorCode);
}
});
Expand Down
7 changes: 6 additions & 1 deletion console/src/main/java/org/zstack/console/ManagementServerConsoleProxyBackend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.zstack.header.message.APIMessage;
import org.zstack.header.message.Message;
import org.zstack.header.message.MessageReply;
import org.zstack.header.host.HypervisorType;
import org.zstack.header.vm.VmInstanceInventory;
import org.zstack.utils.*;
import org.zstack.utils.function.Function;
Expand Down Expand Up @@ -199,7 +200,11 @@ protected ConsoleProxy getConsoleProxy(SessionInventory session, VmInstanceInven
inv.setProxyHostname(mgmtIp);
inv.setAgentIp("127.0.0.1");
inv.setAgentType(getConsoleBackendType());
inv.setToken(session.getUuid() + "_" + vm.getUuid());
String token = session.getUuid() + "_" + vm.getUuid();
if (consoleMgr.getHypervisorConsoleBackend(HypervisorType.valueOf(vm.getHypervisorType())).requireExclusiveConsoleSessionRenewal()) {
token = token + "_" + Platform.getUuid();
}
inv.setToken(token);
inv.setVmInstanceUuid(vm.getUuid());
return new ConsoleProxyBase(inv, getAgentPort());
}
Expand Down
4 changes: 4 additions & 0 deletions header/src/main/java/org/zstack/header/console/ConsoleHypervisorBackend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,8 @@ public interface ConsoleHypervisorBackend {
HypervisorType getConsoleBackendHypervisorType();

void generateConsoleUrl(VmInstanceInventory vm, ReturnValueCompletion<ConsoleUrl> complete);

default boolean requireExclusiveConsoleSessionRenewal() {
return false;
}
}
184 changes: 184 additions & 0 deletions test/src/test/groovy/org/zstack/test/integration/console/ConsoleProxyCase.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,14 @@ import org.zstack.core.db.DatabaseFacade
import org.zstack.core.db.Q
import org.zstack.core.gc.GCStatus
import org.zstack.header.console.ConsoleConstants
import org.zstack.header.console.ConsoleHypervisorBackend
import org.zstack.header.console.ConsoleProxyAgentVO
import org.zstack.header.console.ConsoleProxyCommands
import org.zstack.header.console.ConsoleProxyVO
import org.zstack.header.console.ConsoleProxyVO_
import org.zstack.header.console.ConsoleUrl
import org.zstack.header.core.ReturnValueCompletion
import org.zstack.header.host.HypervisorType
import org.zstack.header.vm.KvmReportVmShutdownFromGuestEventMsg
import org.zstack.sdk.ConsoleInventory
import org.zstack.sdk.ConsoleProxyAgentInventory
Expand All @@ -26,6 +30,9 @@ import org.zstack.testlib.EnvSpec
import org.zstack.testlib.HttpError
import org.zstack.testlib.SubCase
import org.zstack.utils.data.SizeUnit
import org.zstack.utils.gson.JSONObjectUtil

import java.sql.Timestamp

class ConsoleProxyCase extends SubCase {
EnvSpec env
Expand Down Expand Up @@ -127,12 +134,189 @@ class ConsoleProxyCase extends SubCase {
void test() {
env.create {
testSelectConsoleProxyByClientIpVersion()
testExpiredConsoleProxyUsesLegacyRenewal()
testExpiredConsoleProxyUsesExclusiveRenewal()
testConsoleProxyCleanupOnGuestShutdown()
testUpdateConsoleProxyAgent()
testConsoleProxyGC()
}
}

void testExpiredConsoleProxyUsesLegacyRenewal() {
VmInstanceInventory vm = env.inventoryByName("vm")
int deleteProxyCount = 0
int establishProxyCount = 0
ConsoleManagerImpl consoleMgr = bean(ConsoleManagerImpl.class)
def backendsField = ConsoleManagerImpl.class.getDeclaredField("consoleHypervisorBackends")
backendsField.accessible = true
Map<String, ConsoleHypervisorBackend> backends = backendsField.get(consoleMgr)
assert !backends.get(vm.hypervisorType).requireExclusiveConsoleSessionRenewal()

env.afterSimulator(ConsoleConstants.CONSOLE_PROXY_ESTABLISH_PROXY_PATH) { rsp, HttpEntity<String> e ->
ConsoleProxyCommands.EstablishProxyCmd cmd = JSONObjectUtil.toObject(e.body, ConsoleProxyCommands.EstablishProxyCmd.class)
if (cmd.vmUuid == vm.uuid) {
establishProxyCount++
}
return rsp
}
env.afterSimulator(ConsoleConstants.CONSOLE_PROXY_DELETE_PROXY_PATH) { rsp, HttpEntity<String> e ->
ConsoleProxyCommands.DeleteProxyCmd cmd = JSONObjectUtil.toObject(e.body, ConsoleProxyCommands.DeleteProxyCmd.class)
if (cmd.vmUuid == vm.uuid) {
deleteProxyCount++
}
return rsp
}

ConsoleInventory console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory

ConsoleProxyVO vo = Q.New(ConsoleProxyVO.class)
.eq(ConsoleProxyVO_.vmInstanceUuid, vm.uuid)
.find()
String firstToken = console.token
console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory
assert establishProxyCount == 2
assert deleteProxyCount == 0
assert console.token == firstToken

vo.expiredDate = new Timestamp(System.currentTimeMillis() - 1000)
dbf.update(vo)

console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory

assert establishProxyCount == 3
assert deleteProxyCount == 0
assert console.token == "${env.session.uuid}_${vm.uuid}"
}

void testExpiredConsoleProxyUsesExclusiveRenewal() {
VmInstanceInventory vm = env.inventoryByName("vm")
ConsoleProxyVO current = Q.New(ConsoleProxyVO.class)
.eq(ConsoleProxyVO_.vmInstanceUuid, vm.uuid)
.find()
dbf.remove(current)

ConsoleManagerImpl consoleMgr = bean(ConsoleManagerImpl.class)
def backendsField = ConsoleManagerImpl.class.getDeclaredField("consoleHypervisorBackends")
backendsField.accessible = true
Map<String, ConsoleHypervisorBackend> backends = backendsField.get(consoleMgr)
ConsoleHypervisorBackend originalBackend = backends.get(vm.hypervisorType)
backends.put(vm.hypervisorType, new ConsoleHypervisorBackend() {
@Override
HypervisorType getConsoleBackendHypervisorType() {
return originalBackend.getConsoleBackendHypervisorType()
}

@Override
void generateConsoleUrl(org.zstack.header.vm.VmInstanceInventory inventory, ReturnValueCompletion<ConsoleUrl> completion) {
originalBackend.generateConsoleUrl(inventory, completion)
}

@Override
boolean requireExclusiveConsoleSessionRenewal() {
return true
}
})

int deleteProxyCount = 0
int establishProxyCount = 0
boolean failDelete = false
env.afterSimulator(ConsoleConstants.CONSOLE_PROXY_ESTABLISH_PROXY_PATH) { rsp, HttpEntity<String> e ->
ConsoleProxyCommands.EstablishProxyCmd cmd = JSONObjectUtil.toObject(e.body, ConsoleProxyCommands.EstablishProxyCmd.class)
if (cmd.vmUuid == vm.uuid) {
establishProxyCount++
}
return rsp
}
env.afterSimulator(ConsoleConstants.CONSOLE_PROXY_DELETE_PROXY_PATH) { rsp, HttpEntity<String> e ->
ConsoleProxyCommands.DeleteProxyCmd cmd = JSONObjectUtil.toObject(e.body, ConsoleProxyCommands.DeleteProxyCmd.class)
if (cmd.vmUuid == vm.uuid) {
deleteProxyCount++
}
if (failDelete && cmd.vmUuid == vm.uuid) {
throw new HttpError(504, "on purpose")
}
return rsp
}

try {
ConsoleInventory console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory
String firstToken = console.token
String tokenPrefix = "${env.session.uuid}_${vm.uuid}_"
assert firstToken.startsWith(tokenPrefix)
assert firstToken.substring(tokenPrefix.length()) ==~ /[0-9a-f]{32}/

console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory
assert establishProxyCount == 2
assert deleteProxyCount == 0
assert console.token == firstToken

ConsoleProxyVO vo = Q.New(ConsoleProxyVO.class)
.eq(ConsoleProxyVO_.vmInstanceUuid, vm.uuid)
.find()
vo.expiredDate = new Timestamp(System.currentTimeMillis() - 1000)
dbf.update(vo)

console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory

assert establishProxyCount == 3
assert deleteProxyCount == 1
assert console.token.startsWith(tokenPrefix)
assert console.token.substring(tokenPrefix.length()) ==~ /[0-9a-f]{32}/
assert console.token != firstToken

vo = Q.New(ConsoleProxyVO.class)
.eq(ConsoleProxyVO_.vmInstanceUuid, vm.uuid)
.find()
vo.expiredDate = new Timestamp(System.currentTimeMillis() - 1000)
dbf.update(vo)
failDelete = true
int deleteProxyCountBefore = deleteProxyCount
expectError {
requestConsoleAccess {
vmInstanceUuid = vm.uuid
}
}
assert establishProxyCount == 3
assert deleteProxyCount > deleteProxyCountBefore

deleteProxyCountBefore = deleteProxyCount
expectError {
requestConsoleAccess {
vmInstanceUuid = vm.uuid
}
}
assert establishProxyCount == 3
assert deleteProxyCount > deleteProxyCountBefore

failDelete = false
console = requestConsoleAccess {
vmInstanceUuid = vm.uuid
} as ConsoleInventory
assert establishProxyCount == 4
assert console.token.startsWith(tokenPrefix)
} finally {
backends.put(vm.hypervisorType, originalBackend)
ConsoleProxyVO vo = Q.New(ConsoleProxyVO.class)
.eq(ConsoleProxyVO_.vmInstanceUuid, vm.uuid)
.find()
if (vo != null) {
dbf.remove(vo)
}
}
}

void testSelectConsoleProxyByClientIpVersion() {
def selectConsoleProxyHostname = ConsoleManagerImpl.class.getDeclaredMethod("selectConsoleProxyHostname",
String.class, String.class, String.class, String.class, Boolean.TYPE, String.class, String.class, String.class)
Expand Down