diff --git a/build/pom.xml b/build/pom.xml index 5a097bb382a..57d54f39046 100755 --- a/build/pom.xml +++ b/build/pom.xml @@ -631,6 +631,11 @@ hostNetworkInterface ${project.version} + + org.zstack + physicalServer + ${project.version} + org.zstack ovn @@ -977,6 +982,11 @@ hostNetworkInterface ${project.version} + + org.zstack + physicalServer + ${project.version} + org.zstack observabilityServer diff --git a/conf/db/upgrade/V5.5.38.1__schema.sql b/conf/db/upgrade/V5.5.38.1__schema.sql new file mode 100644 index 00000000000..fb1fb2c73ab --- /dev/null +++ b/conf/db/upgrade/V5.5.38.1__schema.sql @@ -0,0 +1,88 @@ +CREATE TABLE IF NOT EXISTS `zstack`.`PhysicalServerVO` ( + `uuid` varchar(32) NOT NULL, + `zoneUuid` varchar(32) DEFAULT NULL, + `poolUuid` varchar(32) DEFAULT NULL, + `serialNumber` varchar(255) NOT NULL, + `createDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + `lastOpDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00' ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukPhysicalServerSerialNumber` (`serialNumber`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`PhysicalServerResourceAssignmentVO` ( + `uuid` varchar(32) NOT NULL, + `serverUuid` varchar(32) NOT NULL, + `resourceType` varchar(32) NOT NULL, + `assignmentType` varchar(64) NOT NULL, + `spec` text NOT NULL, + `specGeneration` bigint NOT NULL DEFAULT 1, + `status` text DEFAULT NULL, + `observedGeneration` bigint DEFAULT NULL, + `observedAt` timestamp NULL DEFAULT NULL, + `createDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', + `lastOpDate` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00' ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukPhysicalServerResourceAssignment` (`serverUuid`, `resourceType`, `assignmentType`), + CONSTRAINT `fkPhysicalServerResourceAssignmentServerUuid` + FOREIGN KEY (`serverUuid`) REFERENCES `zstack`.`PhysicalServerVO` (`uuid`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CALL ADD_COLUMN('HostEO', 'serverUuid', 'VARCHAR(32)', 1, NULL); +CALL ADD_COLUMN('ManagementNodeVO', 'serverUuid', 'VARCHAR(32)', 1, NULL); + +UPDATE `zstack`.`HostEO` +SET `serverUuid` = NULL +WHERE `deleted` IS NOT NULL AND `serverUuid` IS NOT NULL; + +DROP PROCEDURE IF EXISTS addPhysicalServerIdentityUniqueKeys; +DELIMITER $$ +CREATE PROCEDURE addPhysicalServerIdentityUniqueKeys() +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = 'zstack' + AND table_name = 'HostEO' + AND index_name = 'ukHostEOServerUuid' + ) THEN + ALTER TABLE `zstack`.`HostEO` + ADD UNIQUE KEY `ukHostEOServerUuid` (`serverUuid`); + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = 'zstack' + AND table_name = 'ManagementNodeVO' + AND index_name = 'ukManagementNodeVOServerUuid' + ) THEN + ALTER TABLE `zstack`.`ManagementNodeVO` + ADD UNIQUE KEY `ukManagementNodeVOServerUuid` (`serverUuid`); + END IF; +END $$ +DELIMITER ; +CALL addPhysicalServerIdentityUniqueKeys(); +DROP PROCEDURE IF EXISTS addPhysicalServerIdentityUniqueKeys; + +CALL ADD_CONSTRAINT( + 'HostEO', + 'fkHostEOServerUuid', + 'serverUuid', + 'PhysicalServerVO', + 'uuid', + 'SET NULL' +); + +DROP VIEW IF EXISTS `zstack`.`HostVO`; +CREATE VIEW `zstack`.`HostVO` AS + SELECT uuid, zoneUuid, clusterUuid, name, description, managementIp, hypervisorType, + state, status, createDate, lastOpDate, architecture, serverUuid + FROM `zstack`.`HostEO` + WHERE deleted IS NULL; + +CALL ADD_CONSTRAINT( + 'ManagementNodeVO', + 'fkManagementNodeVOServerUuid', + 'serverUuid', + 'PhysicalServerVO', + 'uuid', + 'SET NULL' +); diff --git a/conf/persistence.xml b/conf/persistence.xml index e76141af3ad..371ebd523cc 100755 --- a/conf/persistence.xml +++ b/conf/persistence.xml @@ -229,5 +229,7 @@ org.zstack.network.hostNetworkInterface.PhysicalSwitchVO org.zstack.network.hostNetworkInterface.PhysicalSwitchPortVO org.zstack.header.core.external.service.ExternalServiceConfigurationVO + org.zstack.physicalserver.PhysicalServerVO + org.zstack.physicalserver.PhysicalServerResourceAssignmentVO diff --git a/conf/serviceConfig/physicalServer.xml b/conf/serviceConfig/physicalServer.xml new file mode 100644 index 00000000000..5778fc3384c --- /dev/null +++ b/conf/serviceConfig/physicalServer.xml @@ -0,0 +1,23 @@ + + + physicalServer + PhysicalServerApiInterceptor + + + org.zstack.physicalserver.APIQueryPhysicalServerMsg + query + + + + org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentMsg + query + + + + org.zstack.physicalserver.APIUpdatePhysicalServerCpuAssignmentMsg + + + + org.zstack.physicalserver.APIRefreshPhysicalServerCpuIsolationMsg + + diff --git a/conf/springConfigXml/HostManager.xml b/conf/springConfigXml/HostManager.xml index 8147620c4ae..67605682a22 100755 --- a/conf/springConfigXml/HostManager.xml +++ b/conf/springConfigXml/HostManager.xml @@ -74,4 +74,4 @@ - + diff --git a/conf/springConfigXml/Kvm.xml b/conf/springConfigXml/Kvm.xml index a96965fc01b..4b8a25115ee 100755 --- a/conf/springConfigXml/Kvm.xml +++ b/conf/springConfigXml/Kvm.xml @@ -115,6 +115,24 @@ + + + + + + + + + + + + + + + + + diff --git a/conf/springConfigXml/ManagementNodeManager.xml b/conf/springConfigXml/ManagementNodeManager.xml index ab2d7eefaa1..d04ddd9ff2c 100755 --- a/conf/springConfigXml/ManagementNodeManager.xml +++ b/conf/springConfigXml/ManagementNodeManager.xml @@ -17,4 +17,18 @@ + + + + + + + + + + + + + diff --git a/conf/springConfigXml/physicalServer.xml b/conf/springConfigXml/physicalServer.xml new file mode 100644 index 00000000000..760c009035c --- /dev/null +++ b/conf/springConfigXml/physicalServer.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/springConfigXml/zbs.xml b/conf/springConfigXml/zbs.xml index 2e0234ddc1d..89212209eed 100644 --- a/conf/springConfigXml/zbs.xml +++ b/conf/springConfigXml/zbs.xml @@ -29,4 +29,18 @@ + + + + + + + + + + + + + diff --git a/conf/zstack.xml b/conf/zstack.xml index 17bbdfae090..a194805937e 100755 --- a/conf/zstack.xml +++ b/conf/zstack.xml @@ -46,6 +46,7 @@ + diff --git a/core/src/main/java/org/zstack/core/Platform.java b/core/src/main/java/org/zstack/core/Platform.java index 34df20e3e12..9bb9913dbf7 100755 --- a/core/src/main/java/org/zstack/core/Platform.java +++ b/core/src/main/java/org/zstack/core/Platform.java @@ -120,8 +120,10 @@ public class Platform { }; private static final List UNUSABLE_MACHINE_IDENTITIES = Arrays.asList( "none", + "n/a", "unknown", "not specified", + "to be filled", "to be filled by o.e.m.", "to be filled by oem", "default string", @@ -896,6 +898,18 @@ private static String getManagementServerFingerprint() { return MANAGEMENT_SERVER_FINGERPRINT_ALGORITHM + DigestUtils.sha256Hex(StringUtils.join(identities, "\n")); } + public static String getManagementServerSerialNumber() { + String serialNumber = normalizeMachineSerialNumber( + readMachineIdentity("/sys/class/dmi/id/product_serial")); + return serialNumber == null + ? normalizeMachineSerialNumber(readDmiMachineIdentity("system-serial-number")) + : serialNumber; + } + + public static String normalizeMachineSerialNumber(String serialNumber) { + return normalizeMachineIdentity(serialNumber); + } + private static String readMachineIdentity(String path) { File file = new File(path); if (!file.isFile() || !file.canRead()) { diff --git a/core/src/main/java/org/zstack/core/cpu/LocalCpuTopologyCollector.java b/core/src/main/java/org/zstack/core/cpu/LocalCpuTopologyCollector.java new file mode 100644 index 00000000000..ae3647d5f2f --- /dev/null +++ b/core/src/main/java/org/zstack/core/cpu/LocalCpuTopologyCollector.java @@ -0,0 +1,134 @@ +package org.zstack.core.cpu; + +import org.zstack.core.CoreGlobalProperty; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerNumaNode; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public class LocalCpuTopologyCollector { + private static final Path CPU_ROOT = Paths.get("/sys/devices/system/cpu"); + private static final Path NODE_ROOT = Paths.get("/sys/devices/system/node"); + private volatile PhysicalServerCpuTopology testTopology; + + public PhysicalServerCpuTopology collect() { + if (CoreGlobalProperty.UNIT_TEST_ON && testTopology != null) { + return testTopology; + } + try { + SortedSet online = readCpuSet(CPU_ROOT.resolve("online")); + List nodePaths = nodePaths(); + if (nodePaths.isEmpty()) { + return PhysicalServerCpuTopology.from( + Collections.singletonMap("0", node("0", online, online))); + } + + Map result = new LinkedHashMap<>(); + for (Path nodePath : nodePaths) { + String nodeId = nodePath.getFileName().toString().substring("node".length()); + SortedSet cpus = readCpuSet(nodePath.resolve("cpulist")); + SortedSet nodeOnline = new TreeSet<>(cpus); + nodeOnline.retainAll(online); + if (!nodeOnline.isEmpty()) { + result.put(nodeId, node(nodeId, cpus, nodeOnline)); + } + } + if (result.isEmpty()) { + throw new IllegalArgumentException("CPU_TOPOLOGY_UNAVAILABLE: no online NUMA CPU found"); + } + return PhysicalServerCpuTopology.from(result); + } catch (IOException exception) { + throw new IllegalArgumentException("CPU_TOPOLOGY_UNAVAILABLE: " + exception.getMessage(), exception); + } + } + + public void setTestTopology(PhysicalServerCpuTopology testTopology) { + if (!CoreGlobalProperty.UNIT_TEST_ON) { + throw new IllegalStateException("test topology is only available in unit-test mode"); + } + this.testTopology = testTopology; + } + + public void clearTestTopology() { + testTopology = null; + } + + private List nodePaths() throws IOException { + if (!Files.isDirectory(NODE_ROOT)) { + return Collections.emptyList(); + } + List paths = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(NODE_ROOT, "node[0-9]*")) { + for (Path path : stream) { + if (path.getFileName().toString().matches("node[0-9]+")) { + paths.add(path); + } + } + } + paths.sort(Comparator.comparingInt(path -> Integer.parseInt( + path.getFileName().toString().substring("node".length())))); + return paths; + } + + private PhysicalServerNumaNode node( + String nodeId, SortedSet cpus, SortedSet online) throws IOException { + PhysicalServerNumaNode node = new PhysicalServerNumaNode(); + node.setNodeId(nodeId); + node.setOnlineCpus(strings(online)); + node.setCoreGroups(coreGroups(online)); + return node; + } + + private List> coreGroups(SortedSet online) throws IOException { + Set> groups = new LinkedHashSet<>(); + for (Integer cpu : online) { + Path siblings = CPU_ROOT.resolve(String.format("cpu%s/topology/thread_siblings_list", cpu)); + SortedSet group = readCpuSet(siblings); + group.retainAll(online); + if (group.isEmpty()) { + throw new IllegalArgumentException(String.format( + "CPU_TOPOLOGY_INVALID: CPU[%s] has no online core sibling", cpu)); + } + groups.add(group); + } + List> sorted = new ArrayList<>(groups); + sorted.sort(Comparator.comparingInt(SortedSet::first)); + List> result = new ArrayList<>(); + for (SortedSet group : sorted) { + result.add(strings(group)); + } + return result; + } + + private SortedSet readCpuSet(Path path) throws IOException { + String value = new String(Files.readAllBytes(path), StandardCharsets.US_ASCII).trim(); + if (value.isEmpty()) { + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: empty CPU list at " + path); + } + return PhysicalServerCpuSet.parse(value); + } + + private List strings(Set cpus) { + List result = new ArrayList<>(); + for (Integer cpu : cpus) { + result.add(String.valueOf(cpu)); + } + return result; + } +} diff --git a/core/src/main/java/org/zstack/core/cpu/LocalSharedCpuSetExecutor.java b/core/src/main/java/org/zstack/core/cpu/LocalSharedCpuSetExecutor.java new file mode 100644 index 00000000000..9d191eb64db --- /dev/null +++ b/core/src/main/java/org/zstack/core/cpu/LocalSharedCpuSetExecutor.java @@ -0,0 +1,550 @@ +package org.zstack.core.cpu; + +import org.zstack.core.CoreGlobalProperty; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.SharedCpuSetCommand; +import org.zstack.header.physicalserver.SharedCpuSetResponse; +import org.zstack.header.physicalserver.SharedCpuSetServiceHandle; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class LocalSharedCpuSetExecutor { + private static final Path V2_ROOT = Paths.get("/sys/fs/cgroup"); + private static final Path V1_ROOT = Paths.get("/sys/fs/cgroup/cpuset"); + private static final long COMMAND_TIMEOUT_SECONDS = 30; + + private final AtomicInteger testCalls = new AtomicInteger(); + private volatile boolean testMode; + private volatile SharedCpuSetCommand lastTestCommand; + + public SharedCpuSetResponse apply(SharedCpuSetCommand command) { + if (CoreGlobalProperty.UNIT_TEST_ON && testMode) { + return fakeApply(command); + } + + Backend backend; + try { + backend = backend(); + } catch (SharedCpuSetException exception) { + String reasonCode = reasonCode(exception); + if ("SHARED_CPUSET_UNAVAILABLE".equals(reasonCode)) { + return unavailable(command, reasonCode); + } + throw exception; + } + String desired = command.isEnabled() + ? PhysicalServerCpuSet.normalize(command.getCpuSet()) : ""; + int expected = 0; + int covered = 0; + Set actualSets = new LinkedHashSet<>(); + Set consumerKeys = new LinkedHashSet<>(); + List unresolved = new ArrayList<>(); + + for (SharedCpuSetServiceHandle handle : command.getHandles()) { + try { + Path target = resolve(backend, command.getRoleKey(), handle); + if (target == null) { + continue; + } + expected++; + if (handle.getConsumerKey() != null) { + consumerKeys.add(handle.getConsumerKey()); + } + actualSets.add(applyToGroup(backend, target, desired, command.isEnabled())); + covered++; + } catch (SharedCpuSetException exception) { + expected++; + unresolved.add(unresolvedHandle(handle, reasonCode(exception))); + } + } + + String actual = command.isEnabled() && actualSets.size() == 1 + ? actualSets.iterator().next() : ""; + boolean ready = command.isEnabled() + ? expected > 0 && covered == expected && desired.equals(actual) + : covered == expected; + SharedCpuSetResponse response = new SharedCpuSetResponse(); + response.setBackend(backend.name); + response.setActualCpuSet(actual); + response.setCoveredServiceCount(covered); + response.setExpectedServiceCount(expected); + response.setConsumerKeys(new ArrayList<>(consumerKeys)); + response.setUnresolvedHandles(unresolved); + if (!ready) { + response.setReasonCode(unresolved.isEmpty() + ? "CPUSET_EFFECTIVE_MISMATCH" : "HANDLE_NOT_RESOLVED"); + } + return response; + } + + private SharedCpuSetResponse unavailable( + SharedCpuSetCommand command, String reasonCode) { + Set consumerKeys = new LinkedHashSet<>(); + List unresolved = new ArrayList<>(); + for (SharedCpuSetServiceHandle handle : command.getHandles()) { + if (handle.getConsumerKey() != null) { + consumerKeys.add(handle.getConsumerKey()); + } + unresolved.add(unresolvedHandle(handle, reasonCode)); + } + SharedCpuSetResponse response = new SharedCpuSetResponse(); + response.setActualCpuSet(""); + response.setExpectedServiceCount(command.getHandles().size()); + response.setCoveredServiceCount(0); + response.setConsumerKeys(new ArrayList<>(consumerKeys)); + response.setUnresolvedHandles(unresolved); + response.setReasonCode(reasonCode); + return response; + } + + public void setTestMode(boolean testMode) { + if (!CoreGlobalProperty.UNIT_TEST_ON) { + throw new IllegalStateException("test executor is only available in unit-test mode"); + } + this.testMode = testMode; + this.lastTestCommand = null; + this.testCalls.set(0); + } + + public SharedCpuSetCommand getLastTestCommand() { + return lastTestCommand; + } + + public int getTestCalls() { + return testCalls.get(); + } + + private SharedCpuSetResponse fakeApply(SharedCpuSetCommand command) { + lastTestCommand = command; + testCalls.incrementAndGet(); + Set consumerKeys = new LinkedHashSet<>(); + for (SharedCpuSetServiceHandle handle : command.getHandles()) { + if (handle.getConsumerKey() != null) { + consumerKeys.add(handle.getConsumerKey()); + } + } + SharedCpuSetResponse response = new SharedCpuSetResponse(); + response.setBackend("FAKE_CPUSET"); + response.setActualCpuSet(command.isEnabled() + ? PhysicalServerCpuSet.normalize(command.getCpuSet()) : ""); + response.setExpectedServiceCount(command.getHandles().size()); + response.setCoveredServiceCount(command.getHandles().size()); + response.setConsumerKeys(new ArrayList<>(consumerKeys)); + response.setUnresolvedHandles(Collections.emptyList()); + return response; + } + + private Backend backend() { + Path controllers = V2_ROOT.resolve("cgroup.controllers"); + if (Files.isRegularFile(controllers)) { + String values = read(controllers); + if (values.matches("(?s).*\\bcpuset\\b.*") + || Files.isRegularFile(V2_ROOT.resolve("cpuset.cpus.effective"))) { + return new Backend("CGROUP_V2_CPUSET", V2_ROOT, true); + } + } + if (Files.isRegularFile(V1_ROOT.resolve("cpuset.cpus"))) { + return new Backend("CGROUP_V1_CPUSET", V1_ROOT, false); + } + throw new SharedCpuSetException("SHARED_CPUSET_UNAVAILABLE"); + } + + private Path resolve( + Backend backend, String roleKey, SharedCpuSetServiceHandle handle) { + if (SharedCpuSetServiceHandle.SYSTEMD_UNIT.equals(handle.getHandleType())) { + return resolveSystemd(backend, handle); + } + if (SharedCpuSetServiceHandle.OWNER_PID_FILE.equals(handle.getHandleType())) { + return resolvePidFile(backend, roleKey, handle); + } + throw new SharedCpuSetException("HANDLE_TYPE_UNSUPPORTED"); + } + + private Path resolveSystemd(Backend backend, SharedCpuSetServiceHandle handle) { + Map properties = systemdProperties(handle.getValue()); + if ("not-found".equals(properties.get("LoadState"))) { + if (handle.isOptional()) { + return null; + } + throw new SharedCpuSetException("SYSTEMD_UNIT_NOT_FOUND"); + } + if (!"active".equals(properties.get("ActiveState"))) { + throw new SharedCpuSetException("SYSTEMD_UNIT_NOT_ACTIVE"); + } + String controlGroup = properties.get("ControlGroup"); + if (controlGroup == null || controlGroup.isEmpty()) { + throw new SharedCpuSetException("SYSTEMD_CONTROL_GROUP_MISSING"); + } + Path target = underRoot(backend.root, backend.root.resolve(stripRoot(controlGroup)).normalize()); + if (target.equals(backend.root)) { + throw new SharedCpuSetException("SYSTEMD_CONTROL_GROUP_INVALID"); + } + if (!Files.isDirectory(target)) { + throw new SharedCpuSetException("SYSTEMD_CONTROL_GROUP_NOT_FOUND"); + } + return target; + } + + private Path resolvePidFile( + Backend backend, String roleKey, SharedCpuSetServiceHandle handle) { + Path pidFile = Paths.get(handle.getValue()); + if (!Files.exists(pidFile, LinkOption.NOFOLLOW_LINKS)) { + if (handle.isOptional()) { + return null; + } + throw new SharedCpuSetException("PID_FILE_NOT_FOUND"); + } + try { + PosixFileAttributes attributes = Files.readAttributes( + pidFile, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!attributes.isRegularFile() || Files.isSymbolicLink(pidFile)) { + throw new SharedCpuSetException("PID_FILE_NOT_REGULAR"); + } + Set permissions = attributes.permissions(); + if (permissions.contains(PosixFilePermission.GROUP_WRITE) + || permissions.contains(PosixFilePermission.OTHERS_WRITE)) { + throw new SharedCpuSetException("PID_FILE_PERMISSION_INVALID"); + } + + String content = read(pidFile).trim(); + if (!content.matches("[1-9][0-9]*")) { + throw new SharedCpuSetException("PID_FILE_FORMAT_INVALID"); + } + long parsed = Long.parseLong(content); + if (parsed > Integer.MAX_VALUE) { + throw new SharedCpuSetException("PID_FILE_FORMAT_INVALID"); + } + int pid = (int) parsed; + Path proc = Paths.get("/proc", String.valueOf(pid)); + if (!Files.isDirectory(proc)) { + throw new SharedCpuSetException("PID_NOT_RUNNING"); + } + if (attributes.lastModifiedTime().toMillis() + 1000 < processStartTimeMillis(pid)) { + throw new SharedCpuSetException("PID_FILE_STALE"); + } + PosixFileAttributes procAttributes = Files.readAttributes( + proc, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + if (!"root".equals(attributes.owner().getName()) + && !attributes.owner().equals(procAttributes.owner())) { + throw new SharedCpuSetException("PID_FILE_OWNER_MISMATCH"); + } + String expected = handle.getExpectedCommandToken(); + String command = new String(Files.readAllBytes(proc.resolve("cmdline")), StandardCharsets.UTF_8) + .replace('\0', ' '); + if (expected != null && !command.contains(expected)) { + throw new SharedCpuSetException("PID_COMMAND_MISMATCH"); + } + + String relative = pidCgroup(pid, backend); + if (!"/".equals(relative)) { + Path target = underRoot(backend.root, + backend.root.resolve(stripRoot(relative)).normalize()); + if (!Files.isDirectory(target)) { + throw new SharedCpuSetException("PID_CONTROL_GROUP_NOT_FOUND"); + } + return target; + } + + Path target = backend.root.resolve("zstack-role-" + safeRole(roleKey)); + mkdir(target); + enableV2Path(backend, target); + initializeMems(backend, target); + initializeCpus(backend, target); + write(target.resolve("cgroup.procs"), String.valueOf(pid)); + return target; + } catch (NumberFormatException exception) { + throw new SharedCpuSetException("PID_FILE_FORMAT_INVALID"); + } catch (IOException exception) { + throw new SharedCpuSetException("PID_FILE_READ_FAILED:" + exception.getMessage()); + } + } + + private long processStartTimeMillis(int pid) { + long bootTimeSeconds = -1; + for (String line : read(Paths.get("/proc/stat")).split("\\R")) { + if (line.startsWith("btime ")) { + try { + bootTimeSeconds = Long.parseLong(line.substring("btime ".length()).trim()); + } catch (NumberFormatException exception) { + throw new SharedCpuSetException("PID_START_TIME_UNAVAILABLE"); + } + break; + } + } + String stat = read(Paths.get("/proc", String.valueOf(pid), "stat")); + int commandEnd = stat.lastIndexOf(')'); + if (bootTimeSeconds < 0 || commandEnd < 0) { + throw new SharedCpuSetException("PID_START_TIME_UNAVAILABLE"); + } + String[] fields = stat.substring(commandEnd + 1).trim().split("\\s+"); + if (fields.length <= 19) { + throw new SharedCpuSetException("PID_START_TIME_UNAVAILABLE"); + } + try { + long startTicks = Long.parseLong(fields[19]); + long clockTicks = Long.parseLong(run(null, "getconf", "CLK_TCK").trim()); + if (clockTicks <= 0) { + throw new SharedCpuSetException("PID_START_TIME_UNAVAILABLE"); + } + return bootTimeSeconds * 1000 + startTicks * 1000 / clockTicks; + } catch (NumberFormatException exception) { + throw new SharedCpuSetException("PID_START_TIME_UNAVAILABLE"); + } + } + + private String applyToGroup( + Backend backend, Path target, String desired, boolean enabled) { + enableV2Path(backend, target); + initializeMems(backend, target); + Path cpuFile = target.resolve("cpuset.cpus"); + if (!Files.isRegularFile(cpuFile)) { + throw new SharedCpuSetException("CPUSET_CONTROLLER_NOT_DELEGATED"); + } + String value = enabled ? desired : backend.v2 ? "" : parentCpuSet(target); + String configured = normalizeOptional(read(cpuFile)); + if (!configured.equals(value)) { + write(cpuFile, value.isEmpty() && backend.v2 ? "\n" : value); + } + if (!enabled) { + if (!normalizeOptional(read(cpuFile)).equals(value)) { + throw new SharedCpuSetException("CPUSET_RELEASE_MISMATCH"); + } + return ""; + } + Path effective = target.resolve("cpuset.cpus.effective"); + return normalizeOptional(read(Files.isRegularFile(effective) ? effective : cpuFile)); + } + + private void enableV2Path(Backend backend, Path target) { + if (!backend.v2) { + return; + } + Path current = backend.root; + for (Path part : backend.root.relativize(target)) { + Path child = current.resolve(part); + if (!Files.isRegularFile(child.resolve("cpuset.cpus"))) { + Path control = current.resolve("cgroup.subtree_control"); + if (!Files.isRegularFile(control)) { + throw new SharedCpuSetException("CGROUP_V2_SUBTREE_CONTROL_MISSING"); + } + write(control, "+cpuset"); + } + current = child; + } + } + + private void initializeMems(Backend backend, Path target) { + Path mems = target.resolve("cpuset.mems"); + if (!Files.isRegularFile(mems) || !read(mems).trim().isEmpty()) { + return; + } + Path parent = target.getParent(); + Path source = parent.resolve("cpuset.mems.effective"); + if (!Files.isRegularFile(source)) { + source = parent.resolve("cpuset.mems"); + } + String value = read(source).trim(); + if (value.isEmpty()) { + throw new SharedCpuSetException("CPUSET_MEMS_EMPTY"); + } + write(mems, value); + } + + private void initializeCpus(Backend backend, Path target) { + if (backend.v2) { + return; + } + Path cpus = target.resolve("cpuset.cpus"); + if (read(cpus).trim().isEmpty()) { + write(cpus, parentCpuSet(target)); + } + } + + private String parentCpuSet(Path target) { + Path parent = target.getParent(); + Path source = parent.resolve("cpuset.cpus.effective"); + if (!Files.isRegularFile(source)) { + source = parent.resolve("cpuset.cpus"); + } + String value = normalizeOptional(read(source)); + if (value.isEmpty()) { + throw new SharedCpuSetException("PARENT_CPUSET_EMPTY"); + } + return value; + } + + private String pidCgroup(int pid, Backend backend) { + for (String line : read(Paths.get("/proc", String.valueOf(pid), "cgroup")).split("\\R")) { + String[] parts = line.split(":", 3); + if (parts.length != 3) { + continue; + } + if (backend.v2 && "0".equals(parts[0])) { + return parts[2]; + } + if (!backend.v2) { + for (String controller : parts[1].split(",")) { + if ("cpuset".equals(controller)) { + return parts[2]; + } + } + } + } + throw new SharedCpuSetException("PID_CPUSET_MEMBERSHIP_MISSING"); + } + + private Map systemdProperties(String unit) { + String output = run(null, + "systemctl", "show", unit, + "--property=LoadState", "--property=ActiveState", "--property=ControlGroup"); + Map properties = new HashMap<>(); + for (String line : output.split("\\R")) { + int separator = line.indexOf('='); + if (separator > 0) { + properties.put(line.substring(0, separator), line.substring(separator + 1)); + } + } + return properties; + } + + private void mkdir(Path path) { + if (Files.isDirectory(path)) { + return; + } + run(null, "sudo", "-n", "mkdir", "-p", path.toString()); + } + + private String read(Path path) { + try { + return new String(Files.readAllBytes(path), StandardCharsets.US_ASCII); + } catch (IOException exception) { + throw new SharedCpuSetException("READ_FAILED:" + path + ":" + exception.getMessage()); + } + } + + private void write(Path path, String value) { + run(value.getBytes(StandardCharsets.US_ASCII), + "sudo", "-n", "tee", path.toString()); + } + + private String run(byte[] input, String... command) { + Process process = null; + try { + process = new ProcessBuilder(command).redirectErrorStream(true).start(); + if (input != null) { + try (OutputStream stream = process.getOutputStream()) { + stream.write(input); + } + } else { + process.getOutputStream().close(); + } + if (!process.waitFor(COMMAND_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new SharedCpuSetException("COMMAND_TIMEOUT:" + command[0]); + } + String output = readStream(process.getInputStream()); + if (process.exitValue() != 0) { + throw new SharedCpuSetException( + "COMMAND_FAILED:" + command[0] + ":" + output.trim()); + } + return output; + } catch (IOException exception) { + throw new SharedCpuSetException("COMMAND_FAILED:" + command[0] + ":" + exception.getMessage()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new SharedCpuSetException("COMMAND_INTERRUPTED:" + command[0]); + } finally { + if (process != null) { + process.destroy(); + } + } + } + + private String readStream(InputStream stream) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int count; + while ((count = stream.read(buffer)) != -1) { + output.write(buffer, 0, count); + } + return new String(output.toByteArray(), StandardCharsets.UTF_8); + } + + private String normalizeOptional(String value) { + String trimmed = value == null ? "" : value.trim(); + return trimmed.isEmpty() ? "" : PhysicalServerCpuSet.normalize(trimmed); + } + + private Path underRoot(Path root, Path path) { + if (!path.equals(root) && !path.startsWith(root)) { + throw new SharedCpuSetException("CONTROL_GROUP_PATH_INVALID"); + } + return path; + } + + private String stripRoot(String path) { + return path.startsWith("/") ? path.substring(1) : path; + } + + private String safeRole(String roleKey) { + String value = roleKey == null ? "" : roleKey.replaceAll("[^a-zA-Z0-9_.-]", "-"); + if (value.isEmpty()) { + throw new SharedCpuSetException("ROLE_KEY_INVALID"); + } + return value; + } + + private String reasonCode(SharedCpuSetException exception) { + String message = exception.getMessage(); + if (message == null || message.isEmpty()) { + return "SHARED_CPUSET_ERROR"; + } + int separator = message.indexOf(':'); + return separator > 0 ? message.substring(0, separator) : message; + } + + private String unresolvedHandle(SharedCpuSetServiceHandle handle, String reasonCode) { + String identity = SharedCpuSetServiceHandle.SYSTEMD_UNIT.equals(handle.getHandleType()) + ? handle.getValue() : handle.getHandleType(); + return String.format("%s:%s:%s", handle.getConsumerKey(), identity, reasonCode); + } + + private static class Backend { + private final String name; + private final Path root; + private final boolean v2; + + private Backend(String name, Path root, boolean v2) { + this.name = name; + this.root = root; + this.v2 = v2; + } + } + + public static class SharedCpuSetException extends RuntimeException { + public SharedCpuSetException(String message) { + super(message); + } + } +} diff --git a/header/src/main/java/org/zstack/header/host/HostAO.java b/header/src/main/java/org/zstack/header/host/HostAO.java index 8f7bd58ef95..52c2f11a6f6 100755 --- a/header/src/main/java/org/zstack/header/host/HostAO.java +++ b/header/src/main/java/org/zstack/header/host/HostAO.java @@ -37,6 +37,9 @@ public class HostAO extends ResourceVO { @Column private String architecture; + @Column + private String serverUuid; + @Column @Enumerated(EnumType.STRING) private HostState state; @@ -146,4 +149,12 @@ public String getArchitecture() { public void setArchitecture(String architecture) { this.architecture = architecture; } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } } diff --git a/header/src/main/java/org/zstack/header/host/HostAO_.java b/header/src/main/java/org/zstack/header/host/HostAO_.java index 921e2b5ac64..c6568126895 100755 --- a/header/src/main/java/org/zstack/header/host/HostAO_.java +++ b/header/src/main/java/org/zstack/header/host/HostAO_.java @@ -21,4 +21,5 @@ public class HostAO_ extends ResourceVO_ { public static volatile SingularAttribute createDate; public static volatile SingularAttribute lastOpDate; public static volatile SingularAttribute architecture; + public static volatile SingularAttribute serverUuid; } diff --git a/header/src/main/java/org/zstack/header/host/HostInventory.java b/header/src/main/java/org/zstack/header/host/HostInventory.java index 35958b05003..38d6baf8f93 100755 --- a/header/src/main/java/org/zstack/header/host/HostInventory.java +++ b/header/src/main/java/org/zstack/header/host/HostInventory.java @@ -185,6 +185,8 @@ public class HostInventory implements Serializable { private String architecture; + private String serverUuid; + /** * @desc the time this resource gets created */ @@ -207,6 +209,7 @@ protected HostInventory(HostVO vo) { this.setZoneUuid(vo.getZoneUuid()); this.setClusterUuid(vo.getClusterUuid()); this.setArchitecture(vo.getArchitecture()); + this.setServerUuid(vo.getServerUuid()); if (vo.getCapacity() != null) { this.setTotalCpuCapacity(vo.getCapacity().getTotalCpu()); this.setAvailableCpuCapacity(vo.getCapacity().getAvailableCpu()); @@ -395,6 +398,14 @@ public void setArchitecture(String architecture) { this.architecture = architecture; } + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + public String getIpmiAddress() { return ipmiAddress; } diff --git a/header/src/main/java/org/zstack/header/host/HostNUMANode.java b/header/src/main/java/org/zstack/header/host/HostNUMANode.java index 733a90cc828..39993485e36 100644 --- a/header/src/main/java/org/zstack/header/host/HostNUMANode.java +++ b/header/src/main/java/org/zstack/header/host/HostNUMANode.java @@ -7,6 +7,8 @@ public class HostNUMANode { public List distance; public List cpus; + public List onlineCpus; + public List> coreGroups; public Long free; public Long size; public String nodeID; @@ -36,6 +38,22 @@ public void setCpus(List cpus) { this.cpus = cpus; } + public List getOnlineCpus() { + return onlineCpus; + } + + public void setOnlineCpus(List onlineCpus) { + this.onlineCpus = onlineCpus; + } + + public List> getCoreGroups() { + return coreGroups; + } + + public void setCoreGroups(List> coreGroups) { + this.coreGroups = coreGroups; + } + public Long getFree() { return free; } diff --git a/header/src/main/java/org/zstack/header/host/HostVO.java b/header/src/main/java/org/zstack/header/host/HostVO.java index 319c26750c4..48f0067e435 100755 --- a/header/src/main/java/org/zstack/header/host/HostVO.java +++ b/header/src/main/java/org/zstack/header/host/HostVO.java @@ -80,6 +80,6 @@ protected HostVO(HostVO vo) { this.setCapacity(vo.getCapacity()); this.setIpmi(vo.getIpmi()); this.setHwMonitorStatus(vo.getHwMonitorStatus()); + this.setServerUuid(vo.getServerUuid()); } } - diff --git a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java index 97ce31aec22..f19d8a74c8b 100755 --- a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java +++ b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeInventory.java @@ -17,6 +17,7 @@ public class ManagementNodeInventory { private String hostName; private Timestamp joinDate; private Timestamp heartBeat; + private String serverUuid; public static ManagementNodeInventory valueOf(ManagementNodeVO vo) { ManagementNodeInventory inv = new ManagementNodeInventory(); @@ -24,6 +25,7 @@ public static ManagementNodeInventory valueOf(ManagementNodeVO vo) { inv.setHostName(vo.getHostName()); inv.setJoinDate(vo.getJoinDate()); inv.setUuid(vo.getUuid()); + inv.setServerUuid(vo.getServerUuid()); return inv; } @@ -66,4 +68,12 @@ public Timestamp getJoinDate() { public Timestamp getHeartBeat() { return heartBeat; } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } } diff --git a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java index bd89bcbbf3d..d82cb6ec0e9 100755 --- a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java +++ b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO.java @@ -26,6 +26,9 @@ public class ManagementNodeVO { @Column private int port; + @Column + private String serverUuid; + public String getHostName() { return hostName; } @@ -73,4 +76,12 @@ public void setJoinDate(Timestamp joinDate) { public void setHeartBeat(Timestamp heartBeat) { this.heartBeat = heartBeat; } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } } diff --git a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java index 70035f974cc..81d98c0450f 100755 --- a/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java +++ b/header/src/main/java/org/zstack/header/managementnode/ManagementNodeVO_.java @@ -13,4 +13,5 @@ public class ManagementNodeVO_ { public static volatile SingularAttribute joinDate; public static volatile SingularAttribute heartBeat; public static volatile SingularAttribute state; + public static volatile SingularAttribute serverUuid; } diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuConsumerAdapter.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuConsumerAdapter.java new file mode 100644 index 00000000000..3cc7ad7706e --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuConsumerAdapter.java @@ -0,0 +1,29 @@ +package org.zstack.header.physicalserver; + +import org.zstack.header.core.ReturnValueCompletion; + +import java.util.Set; + +public interface PhysicalServerCpuConsumerAdapter { + String getAssignmentType(); + + default void refreshAssociations() { + } + + default void refreshCapacity(String serverUuid) { + } + + Set getAssociatedServerUuids(); + + PhysicalServerCpuConsumerState getState(String serverUuid); + + void collectTopology( + String serverUuid, + ReturnValueCompletion completion); + + void apply( + String serverUuid, + String consumerUuid, + SharedCpuSetCommand command, + ReturnValueCompletion completion); +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuConsumerState.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuConsumerState.java new file mode 100644 index 00000000000..4e9f8225fbb --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuConsumerState.java @@ -0,0 +1,7 @@ +package org.zstack.header.physicalserver; + +public enum PhysicalServerCpuConsumerState { + AVAILABLE, + UNAVAILABLE, + MISSING +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuSet.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuSet.java new file mode 100644 index 00000000000..d532c7ebea0 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuSet.java @@ -0,0 +1,191 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.math.BigInteger; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public final class PhysicalServerCpuSet { + private PhysicalServerCpuSet() { + } + + public static String normalize(String value) { + return formatRanges(parseRanges(value)); + } + + public static SortedSet parse(String value, Set allowed) { + List ranges = parseRanges(value); + SortedSet cpus = new TreeSet<>(); + for (Range range : ranges) { + long width = (long) range.end - range.start + 1; + if (width > allowed.size()) { + throw new IllegalArgumentException(String.format( + "CPU_SET range[%s-%s] contains CPUs outside the online topology", range.start, range.end)); + } + for (long cursor = range.start; cursor <= range.end; cursor++) { + int cpu = (int) cursor; + if (!allowed.contains(cpu)) { + throw new IllegalArgumentException(String.format( + "CPU[%s] is outside the online topology", cpu)); + } + cpus.add(cpu); + } + } + return cpus; + } + + public static SortedSet parse(String value) { + List ranges = parseRanges(value); + SortedSet cpus = new TreeSet<>(); + for (Range range : ranges) { + long width = (long) range.end - range.start + 1; + if (width > 1048576) { + throw new IllegalArgumentException(String.format( + "CPU_SET range[%s-%s] is too large", range.start, range.end)); + } + for (long cursor = range.start; cursor <= range.end; cursor++) { + cpus.add((int) cursor); + } + } + return cpus; + } + + public static String format(Collection cpus) { + if (cpus == null || cpus.isEmpty()) { + return ""; + } + List sorted = new ArrayList<>(new LinkedHashSet<>(cpus)); + Collections.sort(sorted); + List ranges = new ArrayList<>(); + int start = sorted.get(0); + int end = start; + for (int index = 1; index < sorted.size(); index++) { + int current = sorted.get(index); + if ((long) current == (long) end + 1) { + end = current; + } else { + ranges.add(new Range(start, end)); + start = current; + end = current; + } + } + ranges.add(new Range(start, end)); + return formatRanges(ranges); + } + + public static int count(String value) { + if (value == null || value.trim().isEmpty()) { + return 0; + } + long count = 0; + for (Range range : parseRanges(value)) { + count += (long) range.end - range.start + 1; + if (count > Integer.MAX_VALUE) { + throw new IllegalArgumentException("CPU_SET contains too many CPUs"); + } + } + return (int) count; + } + + public static String union(String left, String right) { + boolean leftEmpty = left == null || left.trim().isEmpty(); + boolean rightEmpty = right == null || right.trim().isEmpty(); + if (leftEmpty && rightEmpty) { + return ""; + } + if (leftEmpty) { + return normalize(right); + } + if (rightEmpty) { + return normalize(left); + } + return normalize(left + "," + right); + } + + private static List parseRanges(String value) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException("CPU_SET cannot be empty"); + } + + List ranges = new ArrayList<>(); + for (String item : value.split(",")) { + String part = item.trim(); + if (part.isEmpty()) { + throw new IllegalArgumentException("CPU_SET contains an empty item"); + } + + String[] range = part.split("-", -1); + if (range.length > 2) { + throw new IllegalArgumentException(String.format("invalid CPU_SET item[%s]", part)); + } + + int start = parseCpu(range[0]); + int end = range.length == 1 ? start : parseCpu(range[1]); + if (start > end) { + throw new IllegalArgumentException(String.format("invalid CPU_SET range[%s]", part)); + } + ranges.add(new Range(start, end)); + } + + ranges.sort(Comparator.comparingInt(range -> range.start)); + List mergedRanges = new ArrayList<>(); + Range merged = ranges.get(0); + for (int index = 1; index < ranges.size(); index++) { + Range current = ranges.get(index); + if ((long) current.start <= (long) merged.end + 1) { + merged.end = Math.max(merged.end, current.end); + } else { + mergedRanges.add(merged); + merged = current; + } + } + mergedRanges.add(merged); + return mergedRanges; + } + + private static String formatRanges(List ranges) { + StringBuilder result = new StringBuilder(); + for (Range range : ranges) { + appendRange(result, range.start, range.end); + } + return result.toString(); + } + + private static int parseCpu(String value) { + String normalized = value.trim(); + if (!normalized.matches("[0-9]+")) { + throw new IllegalArgumentException(String.format("invalid CPU id[%s]", value)); + } + BigInteger cpu = new BigInteger(normalized); + if (cpu.compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException(String.format("CPU id[%s] is too large", value)); + } + return cpu.intValue(); + } + + private static void appendRange(StringBuilder result, int start, int end) { + if (result.length() > 0) { + result.append(','); + } + result.append(start); + if (start != end) { + result.append('-').append(end); + } + } + + private static class Range { + private final int start; + private int end; + + private Range(int start, int end) { + this.start = start; + this.end = end; + } + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuTopology.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuTopology.java new file mode 100644 index 00000000000..41dfbcbedf4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerCpuTopology.java @@ -0,0 +1,129 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public class PhysicalServerCpuTopology { + public static class CoreGroup { + private final String numaId; + private final SortedSet cpus; + + CoreGroup(String numaId, Collection cpus) { + this.numaId = numaId; + this.cpus = new TreeSet<>(cpus); + } + + public String getNumaId() { + return numaId; + } + + public SortedSet getCpus() { + return new TreeSet<>(cpus); + } + + int firstCpu() { + return cpus.first(); + } + } + + private final SortedSet onlineCpus; + private final List coreGroups; + + private PhysicalServerCpuTopology(SortedSet onlineCpus, List coreGroups) { + this.onlineCpus = onlineCpus; + this.coreGroups = coreGroups; + } + + public static PhysicalServerCpuTopology from(Map numaNodes) { + if (numaNodes == null || numaNodes.isEmpty()) { + throw new IllegalArgumentException("CPU_TOPOLOGY_UNAVAILABLE: NUMA topology is empty"); + } + + SortedSet online = new TreeSet<>(); + List groups = new ArrayList<>(); + Set grouped = new HashSet<>(); + Map sortedNodes = new LinkedHashMap<>(); + numaNodes.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> sortedNodes.put(entry.getKey(), entry.getValue())); + + for (Map.Entry entry : sortedNodes.entrySet()) { + PhysicalServerNumaNode node = entry.getValue(); + if (node.getOnlineCpus() == null || node.getCoreGroups() == null || node.getCoreGroups().isEmpty()) { + throw new IllegalArgumentException( + "CPU_TOPOLOGY_UNSUPPORTED: topology provider did not report online CPUs and core sibling groups"); + } + + Set nodeOnline = parseCpuIds(node.getOnlineCpus()); + online.addAll(nodeOnline); + for (List rawGroup : node.getCoreGroups()) { + Set group = parseCpuIds(rawGroup); + if (group.isEmpty() || !nodeOnline.containsAll(group)) { + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: core group is outside its NUMA online CPUs"); + } + for (Integer cpu : group) { + if (!grouped.add(cpu)) { + throw new IllegalArgumentException(String.format( + "CPU_TOPOLOGY_INVALID: CPU[%s] appears in multiple core groups", cpu)); + } + } + groups.add(new CoreGroup(entry.getKey(), group)); + } + } + + if (!grouped.equals(online)) { + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: core groups do not cover every online CPU exactly once"); + } + groups.sort(Comparator.comparingInt(CoreGroup::firstCpu)); + return new PhysicalServerCpuTopology(online, groups); + } + + private static Set parseCpuIds(List values) { + Set result = new HashSet<>(); + for (String value : values) { + if (value == null || !value.matches("[0-9]+")) { + throw new IllegalArgumentException(String.format("CPU_TOPOLOGY_INVALID: invalid CPU id[%s]", value)); + } + BigInteger cpu = new BigInteger(value); + if (cpu.compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0) { + throw new IllegalArgumentException(String.format("CPU_TOPOLOGY_INVALID: CPU id[%s] is too large", value)); + } + result.add(cpu.intValue()); + } + return result; + } + + public SortedSet getOnlineCpus() { + return new TreeSet<>(onlineCpus); + } + + public List getCoreGroups() { + return new ArrayList<>(coreGroups); + } + + public CoreGroup getCpuZeroGroup() { + for (CoreGroup group : coreGroups) { + if (group.cpus.contains(0)) { + return group; + } + } + throw new IllegalArgumentException("CPU_TOPOLOGY_INVALID: online topology does not contain CPU0"); + } + + public String fingerprint() { + List encoded = new ArrayList<>(); + for (CoreGroup group : coreGroups) { + encoded.add(group.numaId + ":" + PhysicalServerCpuSet.format(group.cpus)); + } + return String.join(";", encoded); + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerIdentitySpec.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerIdentitySpec.java new file mode 100644 index 00000000000..0c9c6111ee2 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerIdentitySpec.java @@ -0,0 +1,30 @@ +package org.zstack.header.physicalserver; + +public class PhysicalServerIdentitySpec { + private String serialNumber; + private String zoneUuid; + + public PhysicalServerIdentitySpec() { + } + + public PhysicalServerIdentitySpec(String serialNumber, String zoneUuid) { + this.serialNumber = serialNumber; + this.zoneUuid = zoneUuid; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public String getZoneUuid() { + return zoneUuid; + } + + public void setZoneUuid(String zoneUuid) { + this.zoneUuid = zoneUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerManager.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerManager.java new file mode 100644 index 00000000000..a27b2cab16f --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerManager.java @@ -0,0 +1,27 @@ +package org.zstack.header.physicalserver; + +import org.zstack.header.core.Completion; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +public interface PhysicalServerManager { + Map resolveIdentities(Collection identities); + + void ensureCpuAssignments(Collection serverUuids, String assignmentType); + + default void ensureCpuAssignment(String serverUuid, String assignmentType) { + ensureCpuAssignments(Collections.singleton(serverUuid), assignmentType); + } + + void reconcile(String serverUuid, boolean refreshFacts); + + void reconcileAll(); + + void releaseCpuAssignment( + String serverUuid, + String assignmentType, + String consumerUuid, + Completion completion); +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerNumaNode.java b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerNumaNode.java new file mode 100644 index 00000000000..01725e9b0e6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/PhysicalServerNumaNode.java @@ -0,0 +1,34 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class PhysicalServerNumaNode { + private String nodeId; + private List onlineCpus = new ArrayList<>(); + private List> coreGroups = new ArrayList<>(); + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public List getOnlineCpus() { + return onlineCpus; + } + + public void setOnlineCpus(List onlineCpus) { + this.onlineCpus = onlineCpus; + } + + public List> getCoreGroups() { + return coreGroups; + } + + public void setCoreGroups(List> coreGroups) { + this.coreGroups = coreGroups; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetCommand.java b/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetCommand.java new file mode 100644 index 00000000000..5941f2984ca --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetCommand.java @@ -0,0 +1,52 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class SharedCpuSetCommand { + private String roleKey; + private String cpuSet; + private boolean enabled; + private boolean includeAuxiliaryServices; + private List handles = new ArrayList<>(); + + public String getRoleKey() { + return roleKey; + } + + public void setRoleKey(String roleKey) { + this.roleKey = roleKey; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public boolean isIncludeAuxiliaryServices() { + return includeAuxiliaryServices; + } + + public void setIncludeAuxiliaryServices(boolean includeAuxiliaryServices) { + this.includeAuxiliaryServices = includeAuxiliaryServices; + } + + public List getHandles() { + return handles; + } + + public void setHandles(List handles) { + this.handles = handles; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetResponse.java b/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetResponse.java new file mode 100644 index 00000000000..b66b742817a --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetResponse.java @@ -0,0 +1,70 @@ +package org.zstack.header.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class SharedCpuSetResponse { + private String backend; + private String actualCpuSet; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private List consumerKeys = new ArrayList<>(); + private List unresolvedHandles = new ArrayList<>(); + private String reasonCode; + + public String getBackend() { + return backend; + } + + public void setBackend(String backend) { + this.backend = backend; + } + + public String getActualCpuSet() { + return actualCpuSet; + } + + public void setActualCpuSet(String actualCpuSet) { + this.actualCpuSet = actualCpuSet; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public List getConsumerKeys() { + return consumerKeys; + } + + public void setConsumerKeys(List consumerKeys) { + this.consumerKeys = consumerKeys; + } + + public List getUnresolvedHandles() { + return unresolvedHandles; + } + + public void setUnresolvedHandles(List unresolvedHandles) { + this.unresolvedHandles = unresolvedHandles; + } + + public String getReasonCode() { + return reasonCode; + } + + public void setReasonCode(String reasonCode) { + this.reasonCode = reasonCode; + } +} diff --git a/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetServiceHandle.java b/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetServiceHandle.java new file mode 100644 index 00000000000..451d5868c9b --- /dev/null +++ b/header/src/main/java/org/zstack/header/physicalserver/SharedCpuSetServiceHandle.java @@ -0,0 +1,68 @@ +package org.zstack.header.physicalserver; + +public class SharedCpuSetServiceHandle { + public static final String SYSTEMD_UNIT = "SYSTEMD_UNIT"; + public static final String OWNER_PID_FILE = "OWNER_PID_FILE"; + + private String handleType; + private String value; + private String consumerKey; + private boolean optional; + private String expectedCommandToken; + + public SharedCpuSetServiceHandle() { + } + + public SharedCpuSetServiceHandle( + String handleType, + String value, + String consumerKey, + boolean optional, + String expectedCommandToken) { + this.handleType = handleType; + this.value = value; + this.consumerKey = consumerKey; + this.optional = optional; + this.expectedCommandToken = expectedCommandToken; + } + + public String getHandleType() { + return handleType; + } + + public void setHandleType(String handleType) { + this.handleType = handleType; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getConsumerKey() { + return consumerKey; + } + + public void setConsumerKey(String consumerKey) { + this.consumerKey = consumerKey; + } + + public boolean isOptional() { + return optional; + } + + public void setOptional(boolean optional) { + this.optional = optional; + } + + public String getExpectedCommandToken() { + return expectedCommandToken; + } + + public void setExpectedCommandToken(String expectedCommandToken) { + this.expectedCommandToken = expectedCommandToken; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java index feb4c5c11c7..daedf838732 100644 --- a/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java +++ b/header/src/main/java/org/zstack/header/storage/addon/primary/PrimaryStorageControllerSvc.java @@ -43,6 +43,10 @@ public interface PrimaryStorageControllerSvc { */ void syncConfig(String config); + default void beforePersistAddonInfo(AddonInfo addonInfo, Completion completion) { + completion.success(); + } + void getCapacity(List requiredUrls, ReturnValueCompletion comp); void reportCapacity(ReturnValueCompletion comp); void reportHealthy(ReturnValueCompletion comp); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java index 16ff667f21d..de9e012e022 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostCapacityExtension.java @@ -30,10 +30,14 @@ public class KVMHostCapacityExtension implements KVMHostConnectExtensionPoint, H private ResourceConfigFacade rcf; public void reportCapacity(HostInventory host, Completion completion) { + reportCapacity(host.getUuid(), completion); + } + + public void reportCapacity(String hostUuid, Completion completion) { CheckHostCapacityMsg msg = new CheckHostCapacityMsg(); - msg.setHostUuid(host.getUuid()); + msg.setHostUuid(hostUuid); msg.setTimeout(TimeUnit.MINUTES.toMillis(30)); - bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, host.getUuid()); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); bus.send(msg, new CloudBusCallBack(completion) { @Override public void run(MessageReply rly) { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerAdapter.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerAdapter.java new file mode 100644 index 00000000000..6d75758c855 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerAdapter.java @@ -0,0 +1,718 @@ +package org.zstack.kvm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.compute.host.HostSystemTags; +import org.zstack.compute.host.PostHostConnectExtensionPoint; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorableValue; +import org.zstack.header.host.GetHostNumaTopologyMsg; +import org.zstack.header.host.GetHostNumaTopologyReply; +import org.zstack.header.host.HostAO_; +import org.zstack.header.host.HostAfterConnectedExtensionPoint; +import org.zstack.header.host.HostConstant; +import org.zstack.header.host.HostDeleteExtensionPoint; +import org.zstack.header.host.HostEO; +import org.zstack.header.host.HostException; +import org.zstack.header.host.HostInventory; +import org.zstack.header.host.HostNUMANode; +import org.zstack.header.host.HostStatus; +import org.zstack.header.host.HostVO; +import org.zstack.header.host.HostVO_; +import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerAdapter; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerState; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.physicalserver.PhysicalServerNumaNode; +import org.zstack.header.physicalserver.SharedCpuSetCommand; +import org.zstack.header.physicalserver.SharedCpuSetResponse; +import org.zstack.header.physicalserver.SharedCpuSetServiceHandle; +import org.zstack.header.tag.AbstractSystemTagLifeCycleListener; +import org.zstack.header.tag.SystemTagInventory; +import org.zstack.header.tag.SystemTagVO; +import org.zstack.header.tag.SystemTagVO_; +import org.zstack.utils.TagUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import javax.persistence.Tuple; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.zstack.core.Platform.operr; + +public class KvmPhysicalServerAdapter implements + PhysicalServerCpuConsumerAdapter, + PostHostConnectExtensionPoint, + HostAfterConnectedExtensionPoint, + HostDeleteExtensionPoint, + ManagementNodeReadyExtensionPoint, + Component { + public static final String ASSIGNMENT_TYPE = "COMPUTE_SHARED"; + public static final String APPLY_SHARED_CPUSET_PATH = "/host/sharedcpuset/apply"; + private static final String ERROR_CODE = "ORG_ZSTACK_KVM_10000"; + private static final CLogger logger = Utils.getLogger(KvmPhysicalServerAdapter.class); + private static final List COMPUTE_UNITS = Arrays.asList( + "virtlogd.service", + "zstack-network-agent.service", + "zstack-sharedblock-agent.service"); + private static final List NODE_AUX_UNITS = Arrays.asList( + "node_exporter.service", + "collectd.service", + "collectd_exporter.service", + "process_exporter.service", + "zstack_service_exporter.service", + "pushgateway.service"); + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + @Autowired + private KVMHostCapacityExtension capacityExtension; + + @Override + public String getAssignmentType() { + return ASSIGNMENT_TYPE; + } + + @Override + public Set getAssociatedServerUuids() { + return new LinkedHashSet<>(Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .notNull(HostVO_.serverUuid) + .listValues()); + } + + @Override + public void refreshAssociations() { + backfill(false); + } + + @Override + public PhysicalServerCpuConsumerState getState(String serverUuid) { + Tuple host = Q.New(HostVO.class) + .select(HostVO_.uuid, HostVO_.status) + .eq(HostVO_.serverUuid, serverUuid) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .findTuple(); + if (host == null) { + return PhysicalServerCpuConsumerState.MISSING; + } + return host.get(1, HostStatus.class) == HostStatus.Connected + ? PhysicalServerCpuConsumerState.AVAILABLE + : PhysicalServerCpuConsumerState.UNAVAILABLE; + } + + @Override + public void refreshCapacity(String serverUuid) { + String hostUuid = hostUuid(serverUuid, null); + if (hostUuid == null) { + return; + } + capacityExtension.reportCapacity(hostUuid, new Completion(null) { + @Override + public void success() { + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to refresh capacity for host[uuid:%s] after " + + "physical server CPU assignment changed: %s", + hostUuid, errorCode)); + } + }); + } + + @Override + public void collectTopology( + String serverUuid, + ReturnValueCompletion completion) { + String hostUuid = hostUuid(serverUuid, null); + if (hostUuid == null) { + completion.fail(operr(ERROR_CODE, + "HOST_RELATION_MISSING: physical server[uuid:%s] has no KVM host", + serverUuid)); + return; + } + + GetHostNumaTopologyMsg msg = new GetHostNumaTopologyMsg(); + msg.setHostUuid(hostUuid); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + completeTopology(reply.castReply(), completion); + } + }); + } + + private void completeTopology( + GetHostNumaTopologyReply reply, + ReturnValueCompletion completion) { + completion.success(PhysicalServerCpuTopology.from( + neutralTopology(reply.getNuma()))); + } + + @Override + public void apply( + String serverUuid, + String consumerUuid, + SharedCpuSetCommand command, + ReturnValueCompletion completion) { + String hostUuid = hostUuid(serverUuid, consumerUuid); + if (hostUuid == null) { + completion.fail(operr(ERROR_CODE, + "HOST_RELATION_MISSING: physical server[uuid:%s] has no matching KVM host", + serverUuid)); + return; + } + + SharedCpuSetAgentCommand agentCommand = new SharedCpuSetAgentCommand(); + agentCommand.setRoleKey(command.getRoleKey()); + agentCommand.setCpuSet(command.getCpuSet()); + agentCommand.setEnabled(command.isEnabled()); + agentCommand.setHandles(handles(hostUuid, command.isIncludeAuxiliaryServices())); + + KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); + msg.setPath(APPLY_SHARED_CPUSET_PATH); + msg.setHostUuid(hostUuid); + msg.setCommand(agentCommand); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + ErrorableValue result = + KVMHostAsyncHttpCallReply.unwrap( + reply, SharedCpuSetAgentResponse.class); + if (!result.isSuccess()) { + completion.fail(result.error); + return; + } + completion.success(result.result.toInventory()); + } + }); + } + + @Override + public Flow createPostHostConnectFlow(HostInventory host) { + return new NoRollbackFlow() { + String __name__ = "associate-kvm-host-with-physical-server"; + + @Override + public void run(FlowTrigger trigger, Map data) { + associate(host); + trigger.next(); + } + }; + } + + @Override + public void afterHostConnected(HostInventory host) { + if (physicalServerManager == null) { + return; + } + String serverUuid = Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.uuid, host.getUuid()) + .findValue(); + if (serverUuid != null) { + physicalServerManager.reconcile(serverUuid, true); + } + } + + @Override + public void preDeleteHost(HostInventory inventory) throws HostException { + } + + @Override + public void beforeDeleteHost(HostInventory inventory) { + if (inventory.getServerUuid() == null) { + return; + } + String serverUuid = inventory.getServerUuid(); + org.zstack.core.db.SQL.New(HostEO.class) + .eq(HostAO_.uuid, inventory.getUuid()) + .eq(HostAO_.serverUuid, serverUuid) + .set(HostAO_.serverUuid, null) + .update(); + if (physicalServerManager != null) { + physicalServerManager.reconcile(serverUuid, false); + } + } + + @Override + public void afterDeleteHost(HostInventory inventory) { + } + + @Override + public void managementNodeReady() { + backfill(true); + } + + @Override + public boolean start() { + HostSystemTags.SYSTEM_SERIAL_NUMBER.installLifeCycleListener( + new AbstractSystemTagLifeCycleListener() { + @Override + public void tagCreated(SystemTagInventory tag) { + backfill(true); + } + + @Override + public void tagUpdated( + SystemTagInventory old, SystemTagInventory newTag) { + backfill(true); + } + }); + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Transactional + public void associate(HostInventory host) { + if (physicalServerManager == null + || !KVMConstant.KVM_HYPERVISOR_TYPE.equals(host.getHypervisorType())) { + return; + } + String current = Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.uuid, host.getUuid()) + .findValue(); + if (current != null) { + physicalServerManager.ensureCpuAssignment(current, ASSIGNMENT_TYPE); + return; + } + + String serialNumber = Platform.normalizeMachineSerialNumber( + HostSystemTags.SYSTEM_SERIAL_NUMBER.getTokenByResourceUuid( + host.getUuid(), HostSystemTags.SYSTEM_SERIAL_NUMBER_TOKEN)); + if (serialNumber == null) { + logger.warn(String.format( + "cannot associate host[uuid:%s] with a physical server, " + + "reasonCode[IDENTITY_UNAVAILABLE]", host.getUuid())); + return; + } + String zoneUuid = host.getZoneUuid() == null + ? Q.New(HostVO.class).select(HostVO_.zoneUuid) + .eq(HostVO_.uuid, host.getUuid()).findValue() + : host.getZoneUuid(); + String serverUuid = physicalServerManager.resolveIdentities( + Collections.singletonList( + new PhysicalServerIdentitySpec(serialNumber, zoneUuid))) + .get(serialNumber); + if (serverUuid == null) { + return; + } + clearDeletedHostLink(serverUuid); + if (linkedHost(serverUuid, host.getUuid()) != null) { + return; + } + + Query update = dbf.getEntityManager().createNativeQuery( + "UPDATE IGNORE HostEO SET serverUuid = :serverUuid " + + "WHERE uuid = :hostUuid AND serverUuid IS NULL"); + update.setParameter("serverUuid", serverUuid); + update.setParameter("hostUuid", host.getUuid()); + update.executeUpdate(); + current = Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .eq(HostVO_.uuid, host.getUuid()) + .findValue(); + if (!serverUuid.equals(current)) { + logger.warn(String.format( + "cannot associate host[uuid:%s] with physical server[uuid:%s], " + + "reasonCode[IDENTITY_ASSOCIATION_CONFLICT]", + host.getUuid(), serverUuid)); + return; + } + physicalServerManager.ensureCpuAssignment(serverUuid, ASSIGNMENT_TYPE); + } + + @Transactional + private void backfill(boolean requestReconcile) { + if (physicalServerManager == null) { + return; + } + List hostTuples = Q.New(HostVO.class) + .select(HostVO_.uuid, HostVO_.zoneUuid) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .isNull(HostVO_.serverUuid) + .listTuple(); + if (hostTuples.isEmpty()) { + return; + } + Map zonesByHost = new LinkedHashMap<>(); + for (Tuple tuple : hostTuples) { + zonesByHost.put(tuple.get(0, String.class), tuple.get(1, String.class)); + } + Map> serialsByHost = serialsByHost(zonesByHost.keySet()); + Map> candidatesBySerial = new LinkedHashMap<>(); + for (Map.Entry host : zonesByHost.entrySet()) { + Set serials = serialsByHost.get(host.getKey()); + if (serials == null || serials.size() != 1) { + continue; + } + candidatesBySerial + .computeIfAbsent(serials.iterator().next(), ignored -> new ArrayList<>()) + .add(new HostCandidate(host.getKey(), host.getValue())); + } + + Map candidates = new LinkedHashMap<>(); + List identities = new ArrayList<>(); + for (Map.Entry> candidate : candidatesBySerial.entrySet()) { + if (candidate.getValue().size() != 1) { + logger.warn(String.format( + "cannot backfill host physical server association for serialNumber[%s], " + + "reasonCode[IDENTITY_ALREADY_ASSOCIATED]", + candidate.getKey())); + continue; + } + HostCandidate host = candidate.getValue().get(0); + candidates.put(candidate.getKey(), host); + identities.add(new PhysicalServerIdentitySpec(candidate.getKey(), host.zoneUuid)); + } + Map resolved = physicalServerManager.resolveIdentities(identities); + clearDeletedHostLinks(resolved.values()); + Set used = new HashSet<>(Q.New(HostVO.class) + .select(HostVO_.serverUuid) + .notNull(HostVO_.serverUuid) + .listValues()); + Map links = new LinkedHashMap<>(); + for (Map.Entry candidate : candidates.entrySet()) { + String serverUuid = resolved.get(candidate.getKey()); + if (serverUuid != null && used.add(serverUuid)) { + links.put(candidate.getValue().hostUuid, serverUuid); + } + } + updateHostLinks(links); + + List linked = links.isEmpty() ? Collections.emptyList() : Q.New(HostVO.class) + .select(HostVO_.uuid, HostVO_.serverUuid) + .in(HostVO_.uuid, links.keySet()) + .notNull(HostVO_.serverUuid) + .listTuple(); + Set linkedServers = new LinkedHashSet<>(); + for (Tuple tuple : linked) { + String serverUuid = tuple.get(1, String.class); + linkedServers.add(serverUuid); + } + if (requestReconcile && !linkedServers.isEmpty()) { + physicalServerManager.reconcileAll(); + } + } + + private Map> serialsByHost(Collection hostUuids) { + List tagTuples = Q.New(SystemTagVO.class) + .select(SystemTagVO_.resourceUuid, SystemTagVO_.tag) + .in(SystemTagVO_.resourceUuid, hostUuids) + .eq(SystemTagVO_.resourceType, HostVO.class.getSimpleName()) + .like(SystemTagVO_.tag, TagUtils.tagPatternToSqlPattern( + HostSystemTags.SYSTEM_SERIAL_NUMBER.getTagFormat())) + .listTuple(); + Map> result = new HashMap<>(); + for (Tuple tuple : tagTuples) { + String hostUuid = tuple.get(0, String.class); + String serialNumber = Platform.normalizeMachineSerialNumber( + HostSystemTags.SYSTEM_SERIAL_NUMBER.getTokenByTag( + tuple.get(1, String.class), + HostSystemTags.SYSTEM_SERIAL_NUMBER_TOKEN)); + if (serialNumber != null) { + result.computeIfAbsent(hostUuid, ignored -> new LinkedHashSet<>()) + .add(serialNumber); + } + } + return result; + } + + private void updateHostLinks(Map links) { + if (links.isEmpty()) { + return; + } + StringBuilder sql = new StringBuilder( + "UPDATE IGNORE HostEO SET serverUuid = CASE uuid"); + int index = 0; + for (String ignored : links.keySet()) { + sql.append(" WHEN :host").append(index) + .append(" THEN :server").append(index); + index++; + } + sql.append(" ELSE serverUuid END WHERE serverUuid IS NULL AND uuid IN ("); + for (int i = 0; i < links.size(); i++) { + if (i > 0) { + sql.append(','); + } + sql.append(":host").append(i); + } + sql.append(')'); + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + index = 0; + for (Map.Entry link : links.entrySet()) { + query.setParameter("host" + index, link.getKey()); + query.setParameter("server" + index, link.getValue()); + index++; + } + query.executeUpdate(); + } + + private void clearDeletedHostLinks(Collection serverUuids) { + Set targets = new LinkedHashSet<>(serverUuids); + targets.remove(null); + if (targets.isEmpty()) { + return; + } + StringBuilder sql = new StringBuilder( + "UPDATE HostEO SET serverUuid = NULL " + + "WHERE deleted IS NOT NULL AND serverUuid IN ("); + for (int i = 0; i < targets.size(); i++) { + if (i > 0) { + sql.append(','); + } + sql.append(":server").append(i); + } + sql.append(')'); + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + int index = 0; + for (String serverUuid : targets) { + query.setParameter("server" + index, serverUuid); + index++; + } + query.executeUpdate(); + } + + private void clearDeletedHostLink(String serverUuid) { + Query query = dbf.getEntityManager().createNativeQuery( + "UPDATE HostEO SET serverUuid = NULL " + + "WHERE deleted IS NOT NULL AND serverUuid = :serverUuid"); + query.setParameter("serverUuid", serverUuid); + query.executeUpdate(); + } + + private String linkedHost(String serverUuid, String excludedHostUuid) { + return Q.New(HostVO.class) + .select(HostVO_.uuid) + .eq(HostVO_.serverUuid, serverUuid) + .notEq(HostVO_.uuid, excludedHostUuid) + .findValue(); + } + + private String hostUuid(String serverUuid, String consumerUuid) { + if (consumerUuid != null) { + return Q.New(HostVO.class) + .select(HostVO_.uuid) + .eq(HostVO_.uuid, consumerUuid) + .eq(HostVO_.serverUuid, serverUuid) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .findValue(); + } + return Q.New(HostVO.class) + .select(HostVO_.uuid) + .eq(HostVO_.serverUuid, serverUuid) + .eq(HostVO_.hypervisorType, KVMConstant.KVM_HYPERVISOR_TYPE) + .findValue(); + } + + private Map neutralTopology( + Map topology) { + Map result = new LinkedHashMap<>(); + if (topology == null) { + return result; + } + for (Map.Entry entry : topology.entrySet()) { + PhysicalServerNumaNode node = new PhysicalServerNumaNode(); + node.setNodeId(entry.getKey()); + node.setOnlineCpus(entry.getValue().getOnlineCpus()); + node.setCoreGroups(entry.getValue().getCoreGroups()); + result.put(entry.getKey(), node); + } + return result; + } + + private List handles( + String hostUuid, boolean includeAuxiliaryServices) { + String consumerKey = String.format("host-agent:%s", hostUuid); + List result = new ArrayList<>(); + result.add(new SharedCpuSetServiceHandle( + SharedCpuSetServiceHandle.OWNER_PID_FILE, + "/var/run/zstack/kvmagent.pid", consumerKey, false, "kvmagent")); + for (String unit : COMPUTE_UNITS) { + result.add(new SharedCpuSetServiceHandle( + SharedCpuSetServiceHandle.SYSTEMD_UNIT, + unit, consumerKey, true, null)); + } + if (includeAuxiliaryServices) { + for (String unit : NODE_AUX_UNITS) { + result.add(new SharedCpuSetServiceHandle( + SharedCpuSetServiceHandle.SYSTEMD_UNIT, + unit, consumerKey, true, null)); + } + } + return result; + } + + public static class SharedCpuSetAgentCommand extends KVMAgentCommands.AgentCommand { + private String roleKey; + private String cpuSet; + private boolean enabled; + private List handles = new ArrayList<>(); + + public String getRoleKey() { + return roleKey; + } + + public void setRoleKey(String roleKey) { + this.roleKey = roleKey; + } + + public String getCpuSet() { + return cpuSet; + } + + public void setCpuSet(String cpuSet) { + this.cpuSet = cpuSet; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public List getHandles() { + return handles; + } + + public void setHandles(List handles) { + this.handles = handles; + } + } + + public static class SharedCpuSetAgentResponse extends KVMAgentCommands.AgentResponse { + private String backend; + private String actualCpuSet; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private List consumerKeys = new ArrayList<>(); + private List unresolvedHandles = new ArrayList<>(); + private String reasonCode; + + public String getBackend() { + return backend; + } + + public void setBackend(String backend) { + this.backend = backend; + } + + public String getActualCpuSet() { + return actualCpuSet; + } + + public void setActualCpuSet(String actualCpuSet) { + this.actualCpuSet = actualCpuSet; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public List getConsumerKeys() { + return consumerKeys; + } + + public void setConsumerKeys(List consumerKeys) { + this.consumerKeys = consumerKeys; + } + + public List getUnresolvedHandles() { + return unresolvedHandles; + } + + public void setUnresolvedHandles(List unresolvedHandles) { + this.unresolvedHandles = unresolvedHandles; + } + + public String getReasonCode() { + return reasonCode; + } + + public void setReasonCode(String reasonCode) { + this.reasonCode = reasonCode; + } + + private SharedCpuSetResponse toInventory() { + SharedCpuSetResponse response = new SharedCpuSetResponse(); + response.setBackend(backend); + response.setActualCpuSet(actualCpuSet); + response.setCoveredServiceCount(coveredServiceCount); + response.setExpectedServiceCount(expectedServiceCount); + response.setConsumerKeys(consumerKeys); + response.setUnresolvedHandles(unresolvedHandles); + response.setReasonCode(reasonCode); + return response; + } + } + + private static class HostCandidate { + private final String hostUuid; + private final String zoneUuid; + + private HostCandidate(String hostUuid, String zoneUuid) { + this.hostUuid = hostUuid; + this.zoneUuid = zoneUuid; + } + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerCpuCascadeExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerCpuCascadeExtension.java new file mode 100644 index 00000000000..7585e5d6c83 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmPhysicalServerCpuCascadeExtension.java @@ -0,0 +1,112 @@ +package org.zstack.kvm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.host.HostInventory; +import org.zstack.header.host.HostVO; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class KvmPhysicalServerCpuCascadeExtension extends AbstractAsyncCascadeExtension { + private static final CLogger logger = Utils.getLogger( + KvmPhysicalServerCpuCascadeExtension.class); + private static final String NAME = "PhysicalServerComputeSharedCpuAssignment"; + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (physicalServerManager == null + || !HostVO.class.getSimpleName().equals(action.getParentIssuer()) + || !action.isActionCode( + CascadeConstant.DELETION_DELETE_CODE, + CascadeConstant.DELETION_FORCE_DELETE_CODE)) { + completion.success(); + return; + } + + List hosts = action.getParentIssuerContext(); + if (hosts == null || hosts.isEmpty()) { + completion.success(); + return; + } + boolean forceDelete = action.isActionCode(CascadeConstant.DELETION_FORCE_DELETE_CODE); + List forceDeleteFailures = Collections.synchronizedList(new ArrayList<>()); + + new While<>(hosts).each((host, each) -> { + if (host.getServerUuid() == null) { + each.done(); + return; + } + physicalServerManager.releaseCpuAssignment( + host.getServerUuid(), + KvmPhysicalServerAdapter.ASSIGNMENT_TYPE, + host.getUuid(), + new Completion(each) { + @Override + public void success() { + each.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + if (!forceDelete) { + each.addError(errorCode); + each.done(); + return; + } + forceDeleteFailures.add(String.format( + "host[uuid:%s]: %s", host.getUuid(), errorCode)); + logger.error(String.format( + "failed to release compute shared CPUSet before force deleting " + + "host[uuid:%s], keeping the assignment for recovery: %s", + host.getUuid(), errorCode)); + each.done(); + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!forceDelete && !errorCodeList.getCauses().isEmpty()) { + completion.fail(errorCodeList.getCauses().get(0)); + return; + } + if (!forceDeleteFailures.isEmpty()) { + logger.error(String.format( + "failed to release compute shared CPUSet for %s host(s) during " + + "force deletion; assignments are kept for recovery: %s", + forceDeleteFailures.size(), String.join("; ", forceDeleteFailures))); + } + completion.success(); + } + }); + } + + @Override + public List getEdgeNames() { + return Collections.singletonList(HostVO.class.getSimpleName()); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + return null; + } +} diff --git a/plugin/physicalServer/pom.xml b/plugin/physicalServer/pom.xml new file mode 100644 index 00000000000..3f7f96e718e --- /dev/null +++ b/plugin/physicalServer/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + plugin + org.zstack + 5.5.0 + .. + + physicalServer + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${project.compiler.version} + + groovy-eclipse-compiler + ${project.java.version} + ${project.java.version} + lines,vars,source + true + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy.eclipse.compiler} + + + org.codehaus.groovy + groovy-eclipse-batch + ${groovy.eclipse.batch} + + + + + org.codehaus.mojo + aspectj-maven-plugin + ${aspectj.plugin.version} + + + + compile + test-compile + + + + + ${project.java.version} + ${project.java.version} + ${project.java.version} + true + + + org.springframework + spring-aspects + + + org.zstack + core + + + org.zstack + header + + + + + + + diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsg.java new file mode 100644 index 00000000000..a48d70aa082 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsg.java @@ -0,0 +1,25 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.query.APIQueryMessage; +import org.zstack.header.query.AutoQuery; +import org.zstack.header.rest.RestRequest; + +import java.util.List; + +import static java.util.Arrays.asList; + +@AutoQuery(replyClass = APIQueryPhysicalServerReply.class, inventoryClass = PhysicalServerInventory.class) +@Action(category = PhysicalServerConstant.ACTION_CATEGORY, names = {"read"}) +@RestRequest( + path = "/physical-servers", + optionalPaths = {"/physical-servers/{uuid}"}, + method = HttpMethod.GET, + responseClass = APIQueryPhysicalServerReply.class +) +public class APIQueryPhysicalServerMsg extends APIQueryMessage { + public static List __example__() { + return asList("uuid=" + uuid()); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..7547462f7a2 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerMsgDoc_zh_cn.groovy @@ -0,0 +1,31 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIQueryPhysicalServerReply +import org.zstack.header.query.APIQueryMessage + +doc { + title "查询物理服务器(QueryPhysicalServer)" + + category "物理服务器" + + desc """查询由计算节点或管理节点自动识别并关联的物理服务器。""" + + rest { + request { + url "GET /v1/physical-servers" + url "GET /v1/physical-servers/{uuid}" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIQueryPhysicalServerMsg.class + + desc """""" + + params APIQueryMessage.class + } + + response { + clz APIQueryPhysicalServerReply.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReply.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReply.java new file mode 100644 index 00000000000..a2e50d42f13 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReply.java @@ -0,0 +1,19 @@ +package org.zstack.physicalserver; + +import org.zstack.header.query.APIQueryReply; +import org.zstack.header.rest.RestResponse; + +import java.util.List; + +@RestResponse(allTo = "inventories") +public class APIQueryPhysicalServerReply extends APIQueryReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReplyDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..bd38b3e1595 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.PhysicalServerInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "查询物理服务器的返回" + + ref { + name "inventories" + path "org.zstack.physicalserver.APIQueryPhysicalServerReply.inventories" + desc "物理服务器清单" + type "List" + since "5.5.38" + clz PhysicalServerInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIQueryPhysicalServerReply.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsg.java new file mode 100644 index 00000000000..01ddf6c4694 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsg.java @@ -0,0 +1,28 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.query.APIQueryMessage; +import org.zstack.header.query.AutoQuery; +import org.zstack.header.rest.RestRequest; + +import java.util.List; + +import static java.util.Arrays.asList; + +@AutoQuery( + replyClass = APIQueryPhysicalServerResourceAssignmentReply.class, + inventoryClass = PhysicalServerResourceAssignmentInventory.class +) +@Action(category = PhysicalServerConstant.ACTION_CATEGORY, names = {"read"}) +@RestRequest( + path = "/physical-servers/resource-assignments", + optionalPaths = {"/physical-servers/resource-assignments/{uuid}"}, + method = HttpMethod.GET, + responseClass = APIQueryPhysicalServerResourceAssignmentReply.class +) +public class APIQueryPhysicalServerResourceAssignmentMsg extends APIQueryMessage { + public static List __example__() { + return asList("serverUuid=" + uuid(), "resourceType=CPU"); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..11aa68df519 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentMsgDoc_zh_cn.groovy @@ -0,0 +1,31 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentReply +import org.zstack.header.query.APIQueryMessage + +doc { + title "查询物理服务器资源分配(QueryPhysicalServerResourceAssignment)" + + category "物理服务器" + + desc """查询物理服务器资源分配的期望、实际状态及展示态。""" + + rest { + request { + url "GET /v1/physical-servers/resource-assignments" + url "GET /v1/physical-servers/resource-assignments/{uuid}" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIQueryPhysicalServerResourceAssignmentMsg.class + + desc """""" + + params APIQueryMessage.class + } + + response { + clz APIQueryPhysicalServerResourceAssignmentReply.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReply.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReply.java new file mode 100644 index 00000000000..ac43e18a06c --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReply.java @@ -0,0 +1,19 @@ +package org.zstack.physicalserver; + +import org.zstack.header.query.APIQueryReply; +import org.zstack.header.rest.RestResponse; + +import java.util.List; + +@RestResponse(allTo = "inventories") +public class APIQueryPhysicalServerResourceAssignmentReply extends APIQueryReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReplyDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..83502502dc3 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIQueryPhysicalServerResourceAssignmentReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "查询物理服务器资源分配的返回" + + ref { + name "inventories" + path "org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentReply.inventories" + desc "物理服务器资源分配清单" + type "List" + since "5.5.38" + clz PhysicalServerResourceAssignmentInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIQueryPhysicalServerResourceAssignmentReply.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationEvent.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationEvent.java new file mode 100644 index 00000000000..59c40dbf333 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationEvent.java @@ -0,0 +1,14 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; + +@RestResponse +public class APIRefreshPhysicalServerCpuIsolationEvent extends APIEvent { + public APIRefreshPhysicalServerCpuIsolationEvent() { + } + + public APIRefreshPhysicalServerCpuIsolationEvent(String apiId) { + super(apiId); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationEventDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..1f43f46e194 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationEventDoc_zh_cn.groovy @@ -0,0 +1,23 @@ +package org.zstack.physicalserver + +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "刷新物理服务器CPU隔离的返回" + + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIRefreshPhysicalServerCpuIsolationEvent.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationMsg.java new file mode 100644 index 00000000000..e1739ce38c9 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationMsg.java @@ -0,0 +1,35 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; + +@Action(category = PhysicalServerConstant.ACTION_CATEGORY) +@RestRequest( + path = "/physical-servers/{serverUuid}/cpu-isolation/actions", + method = HttpMethod.PUT, + responseClass = APIRefreshPhysicalServerCpuIsolationEvent.class, + isAction = true +) +public class APIRefreshPhysicalServerCpuIsolationMsg extends APIMessage implements PhysicalServerMessage { + @APIParam(resourceType = PhysicalServerVO.class, operationTarget = true) + private String serverUuid; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public static APIRefreshPhysicalServerCpuIsolationMsg __example__() { + APIRefreshPhysicalServerCpuIsolationMsg msg = + new APIRefreshPhysicalServerCpuIsolationMsg(); + msg.setServerUuid(uuid()); + return msg; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..e72793541b6 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIRefreshPhysicalServerCpuIsolationMsgDoc_zh_cn.groovy @@ -0,0 +1,61 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIRefreshPhysicalServerCpuIsolationEvent + +doc { + title "刷新物理服务器CPU隔离(RefreshPhysicalServerCpuIsolation)" + + category "物理服务器" + + desc """异步触发指定物理服务器的CPU分配事实探测与收敛。接口返回成功仅表示任务已受理，实际结果通过资源分配查询接口查看。""" + + rest { + request { + url "PUT /v1/physical-servers/{serverUuid}/cpu-isolation/actions" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIRefreshPhysicalServerCpuIsolationMsg.class + + desc """""" + + params { + + column { + name "serverUuid" + enclosedIn "refreshPhysicalServerCpuIsolation" + desc "物理服务器UUID" + location "url" + type "String" + optional false + since "5.5.38" + + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + } + } + + response { + clz APIRefreshPhysicalServerCpuIsolationEvent.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentEvent.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentEvent.java new file mode 100644 index 00000000000..0e821267709 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentEvent.java @@ -0,0 +1,24 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; + +@RestResponse(allTo = "inventory") +public class APIUpdatePhysicalServerCpuAssignmentEvent extends APIEvent { + private PhysicalServerResourceAssignmentInventory inventory; + + public APIUpdatePhysicalServerCpuAssignmentEvent() { + } + + public APIUpdatePhysicalServerCpuAssignmentEvent(String apiId) { + super(apiId); + } + + public PhysicalServerResourceAssignmentInventory getInventory() { + return inventory; + } + + public void setInventory(PhysicalServerResourceAssignmentInventory inventory) { + this.inventory = inventory; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentEventDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..c26daff6910 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentEventDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "更新物理服务器CPU分配的返回" + + ref { + name "inventory" + path "org.zstack.physicalserver.APIUpdatePhysicalServerCpuAssignmentEvent.inventory" + desc "已持久化的资源分配清单；实际状态可能仍在异步收敛" + type "PhysicalServerResourceAssignmentInventory" + since "5.5.38" + clz PhysicalServerResourceAssignmentInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "5.5.38" + } + ref { + name "error" + path "org.zstack.physicalserver.APIUpdatePhysicalServerCpuAssignmentEvent.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null",false + type "ErrorCode" + since "5.5.38" + clz ErrorCode.class + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentMsg.java new file mode 100644 index 00000000000..960b8f9ffe6 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentMsg.java @@ -0,0 +1,94 @@ +package org.zstack.physicalserver; + +import org.springframework.http.HttpMethod; +import org.zstack.header.identity.Action; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; + +@Action(category = PhysicalServerConstant.ACTION_CATEGORY) +@RestRequest( + path = "/physical-servers/{serverUuid}/cpu-assignments/{assignmentType}/actions", + method = HttpMethod.PUT, + responseClass = APIUpdatePhysicalServerCpuAssignmentEvent.class, + isAction = true +) +public class APIUpdatePhysicalServerCpuAssignmentMsg extends APIMessage implements PhysicalServerMessage { + @APIParam(resourceType = PhysicalServerVO.class, operationTarget = true) + private String serverUuid; + + @APIParam + private String assignmentType; + + @APIParam(required = false) + private String resourceType = PhysicalServerConstant.CPU_RESOURCE_TYPE; + + @APIParam(required = false) + private Boolean enabled; + + @APIParam(required = false) + private String selectionType; + + @APIParam(required = false) + private String selectionValue; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getAssignmentType() { + return assignmentType; + } + + public void setAssignmentType(String assignmentType) { + this.assignmentType = assignmentType; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public Boolean isEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public String getSelectionType() { + return selectionType; + } + + public void setSelectionType(String selectionType) { + this.selectionType = selectionType; + } + + public String getSelectionValue() { + return selectionValue; + } + + public void setSelectionValue(String selectionValue) { + this.selectionValue = selectionValue; + } + + public static APIUpdatePhysicalServerCpuAssignmentMsg __example__() { + APIUpdatePhysicalServerCpuAssignmentMsg msg = + new APIUpdatePhysicalServerCpuAssignmentMsg(); + msg.setServerUuid(uuid()); + msg.setAssignmentType(PhysicalServerCpuAssignmentType.COMPUTE_SHARED.name()); + msg.setEnabled(true); + msg.setSelectionType(PhysicalServerCpuSelectionType.CPU_SET.name()); + msg.setSelectionValue("0-3"); + return msg; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentMsgDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..566bd1711fa --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/APIUpdatePhysicalServerCpuAssignmentMsgDoc_zh_cn.groovy @@ -0,0 +1,111 @@ +package org.zstack.physicalserver + +import org.zstack.physicalserver.APIUpdatePhysicalServerCpuAssignmentEvent + +doc { + title "更新物理服务器CPU分配(UpdatePhysicalServerCpuAssignment)" + + category "物理服务器" + + desc """更新指定物理服务器的CPU分配期望并异步触发收敛。ZBS_EXCLUSIVE 会影响计算容量；共享分配仅设置服务CPU亲和范围，不扣减计算容量。""" + + rest { + request { + url "PUT /v1/physical-servers/{serverUuid}/cpu-assignments/{assignmentType}/actions" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIUpdatePhysicalServerCpuAssignmentMsg.class + + desc """""" + + params { + + column { + name "serverUuid" + enclosedIn "updatePhysicalServerCpuAssignment" + desc "物理服务器UUID" + location "url" + type "String" + optional false + since "5.5.38" + + } + column { + name "assignmentType" + enclosedIn "updatePhysicalServerCpuAssignment" + desc "分配类型：ZBS_EXCLUSIVE、MANAGEMENT_SHARED或COMPUTE_SHARED" + location "url" + type "String" + optional false + since "5.5.38" + + } + column { + name "resourceType" + enclosedIn "updatePhysicalServerCpuAssignment" + desc "资源类型，当前仅支持CPU，默认CPU" + location "body" + type "String" + optional true + since "5.5.38" + + } + column { + name "enabled" + enclosedIn "updatePhysicalServerCpuAssignment" + desc "是否启用该分配；可单独更新启用状态" + location "body" + type "Boolean" + optional true + since "5.5.38" + + } + column { + name "selectionType" + enclosedIn "updatePhysicalServerCpuAssignment" + desc "选择方式：PERCENTAGE、COUNT或CPU_SET；必须与selectionValue同时填写" + location "body" + type "String" + optional true + since "5.5.38" + + } + column { + name "selectionValue" + enclosedIn "updatePhysicalServerCpuAssignment" + desc "选择值；百分比和数量使用整数，CPU集合使用如0-3,8-11的格式" + location "body" + type "String" + optional true + since "5.5.38" + + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.5.38" + + } + } + } + + response { + clz APIUpdatePhysicalServerCpuAssignmentEvent.class + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/FakeZbsCpuIsolationProvider.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/FakeZbsCpuIsolationProvider.java new file mode 100644 index 00000000000..46706f56af1 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/FakeZbsCpuIsolationProvider.java @@ -0,0 +1,291 @@ +package org.zstack.physicalserver; + +import org.zstack.core.CoreGlobalProperty; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; + +import java.sql.Timestamp; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.zstack.core.Platform.operr; + +public class FakeZbsCpuIsolationProvider implements ZbsCpuIsolationProvider { + private final Map facts = new ConcurrentHashMap<>(); + private final Map revisionFences = new ConcurrentHashMap<>(); + private final AtomicInteger queryCalls = new AtomicInteger(); + private final AtomicInteger updateCalls = new AtomicInteger(); + private final AtomicBoolean holdNextQuery = new AtomicBoolean(); + private final AtomicBoolean holdNextUpdate = new AtomicBoolean(); + private final AtomicReference heldQuery = new AtomicReference<>(); + private final AtomicReference heldUpdate = new AtomicReference<>(); + private final AtomicLong lastUpdateRevision = new AtomicLong(); + private volatile boolean enabled; + private volatile boolean autoApply; + + @Override + public String getProviderType() { + return "FAKE"; + } + + @Override + public boolean isAvailable(ZbsNodeRef nodeRef) { + return CoreGlobalProperty.UNIT_TEST_ON && enabled; + } + + @Override + public boolean supportsMonotonicRevisionFence() { + return true; + } + + @Override + public void query(ZbsNodeRef nodeRef, ReturnValueCompletion completion) { + queryCalls.incrementAndGet(); + if (!isAvailable(nodeRef)) { + completion.fail(operr(PhysicalServerConstant.ERROR_CODE, + "FAKE_ZBS_PROVIDER_DISABLED: fake provider never becomes production ready")); + return; + } + ZbsCpuIsolationFact fact = facts.get(nodeRef.getServerUuid()); + if (fact == null) { + completion.fail(operr(PhysicalServerConstant.ERROR_CODE, + "FAKE_ZBS_FACT_MISSING: no fake fact for physical server[uuid:%s]", nodeRef.getServerUuid())); + return; + } + ZbsCpuIsolationFact response = copy(fact); + if (holdNextQuery.compareAndSet(true, false)) { + heldQuery.set(new HeldQuery(completion, response)); + return; + } + completion.success(response); + } + + @Override + public void update(ZbsNodeRef nodeRef, ZbsCpuIsolationUpdate update, Completion completion) { + updateCalls.incrementAndGet(); + lastUpdateRevision.set(update.getRevision()); + if (!isAvailable(nodeRef)) { + completion.fail(operr(PhysicalServerConstant.ERROR_CODE, + "FAKE_ZBS_PROVIDER_DISABLED: fake provider never becomes production ready")); + return; + } + AtomicReference rejected = new AtomicReference<>(); + revisionFences.compute(nodeRef.getServerUuid(), (ignored, current) -> + acceptUpdate(nodeRef, update, current, rejected)); + if (rejected.get() != null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "%s", rejected.get())); + return; + } + if (holdNextUpdate.compareAndSet(true, false)) { + heldUpdate.set(new HeldUpdate(nodeRef, update, completion)); + return; + } + if (autoApply) { + applyUpdate(nodeRef, update); + } + completion.success(); + } + + public void enable() { + enabled = true; + } + + public void disable() { + enabled = false; + autoApply = false; + facts.clear(); + revisionFences.clear(); + queryCalls.set(0); + updateCalls.set(0); + holdNextQuery.set(false); + holdNextUpdate.set(false); + heldQuery.set(null); + heldUpdate.set(null); + lastUpdateRevision.set(0); + } + + public void setAutoApply(boolean autoApply) { + this.autoApply = autoApply; + } + + public void setFact(String serverUuid, ZbsCpuIsolationFact fact) { + facts.put(serverUuid, copy(fact)); + } + + public int getQueryCalls() { + return queryCalls.get(); + } + + public int getUpdateCalls() { + return updateCalls.get(); + } + + public void holdNextQuery() { + holdNextQuery.set(true); + } + + public boolean hasHeldQuery() { + return heldQuery.get() != null; + } + + public void completeHeldQuery() { + HeldQuery held = heldQuery.getAndSet(null); + if (held != null) { + held.completion.success(held.fact); + } + } + + public void holdNextUpdate() { + holdNextUpdate.set(true); + } + + public boolean hasHeldUpdate() { + return heldUpdate.get() != null; + } + + public void completeHeldUpdate() { + HeldUpdate held = heldUpdate.getAndSet(null); + if (held != null) { + RevisionFence fence = revisionFences.get(held.nodeRef.getServerUuid()); + if (autoApply && fence != null && fence.matches(held.update)) { + applyUpdate(held.nodeRef, held.update); + } + held.completion.success(); + } + } + + public long getLastUpdateRevision() { + return lastUpdateRevision.get(); + } + + private void applyUpdate(ZbsNodeRef nodeRef, ZbsCpuIsolationUpdate update) { + ZbsCpuIsolationFact fact = new ZbsCpuIsolationFact(); + fact.setCapabilityVersion("1"); + fact.setIsolationState( + "DISABLE".equals(update.getOperation()) ? "DISABLED" : "READY"); + fact.setConfiguredCpuSet(update.getDesiredCpuSet()); + fact.setEffectiveCpuSet(update.getDesiredCpuSet()); + fact.setExpectedServiceCount(1); + fact.setCoveredServiceCount(1); + fact.setServiceReady(true); + fact.setOwnerAssignmentUuid(update.getAssignmentUuid()); + fact.setOwnerGeneration(String.valueOf(update.getRevision())); + fact.setObservedAt(new Timestamp(System.currentTimeMillis())); + facts.put(nodeRef.getServerUuid(), fact); + } + + private ZbsCpuIsolationFact copy(ZbsCpuIsolationFact source) { + ZbsCpuIsolationFact copy = new ZbsCpuIsolationFact(); + copy.setCapabilityVersion(source.getCapabilityVersion()); + copy.setIsolationState(source.getIsolationState()); + copy.setConfiguredCpuSet(source.getConfiguredCpuSet()); + copy.setEffectiveCpuSet(source.getEffectiveCpuSet()); + copy.setCoveredServiceCount(source.getCoveredServiceCount()); + copy.setExpectedServiceCount(source.getExpectedServiceCount()); + copy.setServiceReady(source.isServiceReady()); + copy.setOwnerAssignmentUuid(source.getOwnerAssignmentUuid()); + copy.setOwnerGeneration(source.getOwnerGeneration()); + copy.setObservedAt(source.getObservedAt()); + copy.setReasonCode(source.getReasonCode()); + return copy; + } + + private RevisionFence acceptUpdate( + ZbsNodeRef nodeRef, + ZbsCpuIsolationUpdate update, + RevisionFence current, + AtomicReference rejected) { + if (update.getAssignmentUuid() == null || update.getAssignmentUuid().isEmpty()) { + rejected.set("OWNER_ASSIGNMENT_MISSING: assignmentUuid is required"); + return current; + } + RevisionFence requested = new RevisionFence(update); + if (current == null) { + return requested; + } + if (current.assignmentUuid.equals(requested.assignmentUuid)) { + if (requested.revision < current.revision) { + rejected.set(String.format( + "OWNER_REVISION_STALE: revision[%s] is older than the highest accepted revision[%s]", + requested.revision, current.revision)); + return current; + } + if (requested.revision == current.revision + && !current.payload.equals(requested.payload)) { + rejected.set(String.format( + "OWNER_REVISION_CONFLICT: revision[%s] was accepted with another payload", + requested.revision)); + return current; + } + return requested.revision == current.revision ? current : requested; + } + + ZbsCpuIsolationFact fact = facts.get(nodeRef.getServerUuid()); + boolean previousReleased = fact != null + && current.assignmentUuid.equals(fact.getOwnerAssignmentUuid()) + && "DISABLED".equals(fact.getIsolationState()) + && (fact.getEffectiveCpuSet() == null || fact.getEffectiveCpuSet().isEmpty()); + if (!previousReleased || !"SET".equals(requested.operation)) { + rejected.set(String.format( + "OWNER_ASSIGNMENT_STALE: assignmentUuid[%s] cannot replace active assignmentUuid[%s]", + requested.assignmentUuid, current.assignmentUuid)); + return current; + } + return requested; + } + + private static class HeldQuery { + private final ReturnValueCompletion completion; + private final ZbsCpuIsolationFact fact; + + private HeldQuery( + ReturnValueCompletion completion, + ZbsCpuIsolationFact fact) { + this.completion = completion; + this.fact = fact; + } + } + + private static class HeldUpdate { + private final ZbsNodeRef nodeRef; + private final ZbsCpuIsolationUpdate update; + private final Completion completion; + + private HeldUpdate( + ZbsNodeRef nodeRef, + ZbsCpuIsolationUpdate update, + Completion completion) { + this.nodeRef = nodeRef; + this.update = update; + this.completion = completion; + } + } + + private static class RevisionFence { + private final String assignmentUuid; + private final long revision; + private final String operation; + private final String payload; + + private RevisionFence(ZbsCpuIsolationUpdate update) { + assignmentUuid = update.getAssignmentUuid(); + revision = update.getRevision(); + operation = update.getOperation(); + payload = String.format("%s|%s|%s", + update.getOperation(), update.getDesiredCpuSet(), update.getPlanHash()); + } + + private boolean matches(ZbsCpuIsolationUpdate update) { + return assignmentUuid.equals(update.getAssignmentUuid()) + && revision == update.getRevision() + && payload.equals(String.format("%s|%s|%s", + update.getOperation(), update.getDesiredCpuSet(), update.getPlanHash())); + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PackageInfo.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PackageInfo.java new file mode 100644 index 00000000000..79a1fe224d5 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PackageInfo.java @@ -0,0 +1,13 @@ +package org.zstack.physicalserver; + +import org.zstack.header.PackageAPIInfo; + +@PackageAPIInfo( + APICategoryName = "物理服务器", + permissions = { + PackageAPIInfo.PERMISSION_COMMUNITY_AVAILABLE, + PackageAPIInfo.PERMISSION_ZSV_BASIC_AVAILABLE + } +) +public class PackageInfo { +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerApiInterceptor.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerApiInterceptor.java new file mode 100644 index 00000000000..f2da1aef103 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerApiInterceptor.java @@ -0,0 +1,105 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.header.apimediator.ApiMessageInterceptionException; +import org.zstack.header.apimediator.ApiMessageInterceptor; +import org.zstack.header.message.APIMessage; + +import static org.zstack.core.Platform.argerr; + +public class PhysicalServerApiInterceptor implements ApiMessageInterceptor { + @Autowired + private CloudBus bus; + @Autowired + private PluginRegistry pluginRgty; + + @Override + public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionException { + if (msg instanceof APIUpdatePhysicalServerCpuAssignmentMsg) { + validate((APIUpdatePhysicalServerCpuAssignmentMsg) msg); + } + + if (msg instanceof PhysicalServerMessage) { + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + } + return msg; + } + + private void validate(APIUpdatePhysicalServerCpuAssignmentMsg msg) { + PhysicalServerResourceAssignmentHandler handler = null; + for (PhysicalServerResourceAssignmentHandler candidate : + pluginRgty.getExtensionList(PhysicalServerResourceAssignmentHandler.class)) { + if (!candidate.getResourceType().equals(msg.getResourceType())) { + continue; + } + if (handler != null) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_TYPE_AMBIGUOUS: multiple handlers registered for resourceType[%s]", + msg.getResourceType())); + } + handler = candidate; + } + if (handler == null || !PhysicalServerConstant.CPU_RESOURCE_TYPE.equals( + handler.getResourceType())) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "RESOURCE_TYPE_NOT_SUPPORTED: resourceType[%s] is not registered", msg.getResourceType())); + } + + if (!handler.supportsAssignmentType(msg.getAssignmentType())) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "ASSIGNMENT_TYPE_NOT_SUPPORTED: assignmentType[%s] is not registered", msg.getAssignmentType())); + } + PhysicalServerCpuAssignmentType assignmentType = PhysicalServerCpuAssignmentType.valueOf(msg.getAssignmentType()); + + if ((msg.getSelectionType() == null) != (msg.getSelectionValue() == null)) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "selectionType and selectionValue must be specified together")); + } + + if (msg.isEnabled() == null && msg.getSelectionType() == null) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "CPU_ASSIGNMENT_UPDATE_EMPTY: enabled or selection must be specified")); + } + + if (msg.getSelectionType() == null) { + return; + } + + if (!isSelectionType(msg.getSelectionType())) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, + "CPU_SELECTION_TYPE_NOT_SUPPORTED: selectionType[%s] is not registered", msg.getSelectionType())); + } + PhysicalServerCpuSelectionType selectionType = PhysicalServerCpuSelectionType.valueOf(msg.getSelectionType()); + + if (assignmentType == PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE + && selectionType == PhysicalServerCpuSelectionType.COUNT) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, "ZBS_EXCLUSIVE does not support COUNT selection")); + } + if (assignmentType != PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE + && selectionType == PhysicalServerCpuSelectionType.PERCENTAGE) { + throw new ApiMessageInterceptionException(argerr( + PhysicalServerConstant.ERROR_CODE, "shared CPU assignment does not support PERCENTAGE selection")); + } + } + + private boolean isSelectionType(String value) { + for (PhysicalServerCpuSelectionType type : PhysicalServerCpuSelectionType.values()) { + if (type.name().equals(value)) { + return true; + } + } + return false; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerAssignmentRepository.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerAssignmentRepository.java new file mode 100644 index 00000000000..a91f566d31d --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerAssignmentRepository.java @@ -0,0 +1,407 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.core.Platform; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.utils.Utils; +import org.zstack.utils.gson.JSONObjectUtil; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.Collection; +import java.util.List; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.ORG_ZSTACK_CORE_10000; + +public class PhysicalServerAssignmentRepository { + private static final CLogger logger = Utils.getLogger(PhysicalServerAssignmentRepository.class); + + @Autowired + private DatabaseFacade dbf; + + @Transactional + public PhysicalServerResourceAssignmentVO ensureDefault( + String serverUuid, PhysicalServerCpuAssignmentType assignmentType) { + PhysicalServerResourceAssignmentVO existing = find(serverUuid, assignmentType); + if (existing != null) { + return existing; + } + + Query insert = dbf.getEntityManager().createNativeQuery( + "INSERT IGNORE INTO PhysicalServerResourceAssignmentVO " + + "(uuid, serverUuid, resourceType, assignmentType, spec, specGeneration, createDate, lastOpDate) " + + "VALUES (:uuid, :serverUuid, :resourceType, :assignmentType, :spec, 1, NOW(), NOW())"); + insert.setParameter("uuid", Platform.getUuid()); + insert.setParameter("serverUuid", serverUuid); + insert.setParameter("resourceType", PhysicalServerConstant.CPU_RESOURCE_TYPE); + insert.setParameter("assignmentType", assignmentType.name()); + insert.setParameter("spec", JSONObjectUtil.toJsonString(defaultSpec(assignmentType))); + insert.executeUpdate(); + return find(serverUuid, assignmentType); + } + + @Transactional + public void ensureDefaults( + Collection serverUuids, PhysicalServerCpuAssignmentType assignmentType) { + if (serverUuids == null || serverUuids.isEmpty()) { + return; + } + Query insert = dbf.getEntityManager().createNativeQuery( + "INSERT IGNORE INTO PhysicalServerResourceAssignmentVO " + + "(uuid, serverUuid, resourceType, assignmentType, spec, specGeneration, createDate, lastOpDate) " + + "SELECT REPLACE(UUID(), '-', ''), p.uuid, :resourceType, :assignmentType, :spec, 1, NOW(), NOW() " + + "FROM PhysicalServerVO p WHERE p.uuid IN (:serverUuids)"); + insert.setParameter("serverUuids", serverUuids); + insert.setParameter("resourceType", PhysicalServerConstant.CPU_RESOURCE_TYPE); + insert.setParameter("assignmentType", assignmentType.name()); + insert.setParameter("spec", JSONObjectUtil.toJsonString(defaultSpec(assignmentType))); + insert.executeUpdate(); + } + + public PhysicalServerResourceAssignmentVO updateCpu(APIUpdatePhysicalServerCpuAssignmentMsg msg) { + PhysicalServerCpuAssignmentType assignmentType = + PhysicalServerCpuAssignmentType.valueOf(msg.getAssignmentType()); + PhysicalServerResourceAssignmentVO current = requireAssignment( + msg.getServerUuid(), assignmentType); + PhysicalServerCpuAssignmentSpec spec = updatedSpec(current, msg); + String newSpec = JSONObjectUtil.toJsonString(spec); + + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a " + + "set a.spec = :spec, a.specGeneration = a.specGeneration + 1 " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("spec", newSpec) + .param("uuid", current.getUuid()) + .param("generation", current.getSpecGeneration()) + .execute(); + if (updated == 0) { + throw new OperationFailureException(operr( + ORG_ZSTACK_CORE_10000, + "CPU assignment on physical server[uuid:%s] changed concurrently, retry the request", + msg.getServerUuid() + )); + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logger.info(String.format( + "physical server resource assignment spec updated: " + + "serverUuid[%s], resourceType[%s], assignmentType[%s], " + + "accountUuid[%s], oldGeneration[%s], newGeneration[%s], " + + "oldSpec[%s], newSpec[%s]", + current.getServerUuid(), current.getResourceType(), current.getAssignmentType(), + msg.getSession() == null ? null : msg.getSession().getAccountUuid(), + current.getSpecGeneration(), result.getSpecGeneration(), + current.getSpec(), result.getSpec())); + return result; + } + + public PhysicalServerCpuAssignmentSpec previewCpuUpdate( + APIUpdatePhysicalServerCpuAssignmentMsg msg) { + PhysicalServerCpuAssignmentType assignmentType = + PhysicalServerCpuAssignmentType.valueOf(msg.getAssignmentType()); + return updatedSpec(requireAssignment(msg.getServerUuid(), assignmentType), msg); + } + + public PhysicalServerResourceAssignmentVO prepareZbsRelease(String serverUuid) { + PhysicalServerResourceAssignmentVO current = find( + serverUuid, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + if (current == null) { + return null; + } + + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + current.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (spec.isCascadeReleaseRequested()) { + return current; + } + + spec.setCascadeReleaseRequested(true); + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a " + + "set a.spec = :spec, a.specGeneration = a.specGeneration + 1 " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("spec", JSONObjectUtil.toJsonString(spec)) + .param("uuid", current.getUuid()) + .param("generation", current.getSpecGeneration()) + .execute(); + if (updated == 0) { + throw new OperationFailureException(operr( + ORG_ZSTACK_CORE_10000, + "CPU assignment on physical server[uuid:%s] changed concurrently, retry deleting the ZBS relationship", + serverUuid + )); + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logger.info(String.format( + "physical server ZBS CPU isolation Cascade release requested: " + + "serverUuid[%s], assignmentUuid[%s], oldGeneration[%s], newGeneration[%s]", + serverUuid, current.getUuid(), current.getSpecGeneration(), result.getSpecGeneration())); + return result; + } + + public PhysicalServerResourceAssignmentVO cancelZbsRelease(String serverUuid) { + PhysicalServerResourceAssignmentVO current = find( + serverUuid, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + if (current == null) { + return null; + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + current.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (!spec.isCascadeReleaseRequested()) { + return current; + } + spec.setCascadeReleaseRequested(false); + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a " + + "set a.spec = :spec, a.specGeneration = a.specGeneration + 1 " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("spec", JSONObjectUtil.toJsonString(spec)) + .param("uuid", current.getUuid()) + .param("generation", current.getSpecGeneration()) + .execute(); + if (updated == 0) { + throw new OperationFailureException(operr( + ORG_ZSTACK_CORE_10000, + "CPU assignment on physical server[uuid:%s] changed while canceling Cascade release", + serverUuid)); + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logger.info(String.format( + "physical server ZBS CPU isolation Cascade release canceled: " + + "serverUuid[%s], assignmentUuid[%s], oldGeneration[%s], newGeneration[%s]", + serverUuid, current.getUuid(), current.getSpecGeneration(), result.getSpecGeneration())); + return result; + } + + public boolean deleteIfGeneration(String uuid, long generation) { + boolean deleted = SQL.New( + "delete from PhysicalServerResourceAssignmentVO a " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("uuid", uuid) + .param("generation", generation) + .execute() == 1; + if (deleted) { + logger.info(String.format( + "physical server resource assignment deleted after owner release: " + + "assignmentUuid[%s], generation[%s]", uuid, generation)); + } + return deleted; + } + + public PhysicalServerResourceAssignmentVO find( + String serverUuid, PhysicalServerCpuAssignmentType assignmentType) { + return Q.New(PhysicalServerResourceAssignmentVO.class) + .eq(PhysicalServerResourceAssignmentVO_.serverUuid, serverUuid) + .eq(PhysicalServerResourceAssignmentVO_.resourceType, PhysicalServerConstant.CPU_RESOURCE_TYPE) + .eq(PhysicalServerResourceAssignmentVO_.assignmentType, assignmentType.name()) + .find(); + } + + public List listCpuAssignments() { + return Q.New(PhysicalServerResourceAssignmentVO.class) + .eq(PhysicalServerResourceAssignmentVO_.resourceType, + PhysicalServerConstant.CPU_RESOURCE_TYPE) + .list(); + } + + public PhysicalServerResourceAssignmentVO updatePlan( + PhysicalServerResourceAssignmentVO current, PhysicalServerCpuPlan plan) { + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + current.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (equals(spec.getDesiredCpuSet(), plan.getDesiredCpuSet()) + && equals(spec.getPlanHash(), plan.getPlanHash())) { + return current; + } + spec.setDesiredCpuSet(plan.getDesiredCpuSet()); + spec.setPlanHash(plan.getPlanHash()); + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a " + + "set a.spec = :spec, a.specGeneration = a.specGeneration + 1 " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("spec", JSONObjectUtil.toJsonString(spec)) + .param("uuid", current.getUuid()) + .param("generation", current.getSpecGeneration()) + .execute(); + if (updated == 0) { + return null; + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logger.info(String.format( + "physical server resource assignment plan updated: " + + "serverUuid[%s], resourceType[%s], assignmentType[%s], " + + "oldGeneration[%s], newGeneration[%s], oldSpec[%s], newSpec[%s]", + current.getServerUuid(), current.getResourceType(), current.getAssignmentType(), + current.getSpecGeneration(), result.getSpecGeneration(), + current.getSpec(), result.getSpec())); + return result; + } + + public PhysicalServerResourceAssignmentVO updateStatus( + PhysicalServerResourceAssignmentVO current, + PhysicalServerCpuAssignmentStatus status, + Timestamp observedAt) { + String statusJson = JSONObjectUtil.toJsonString(status); + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a set a.status = :status, " + + "a.observedGeneration = :generation, a.observedAt = :observedAt " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("status", statusJson) + .param("generation", current.getSpecGeneration()) + .param("observedAt", observedAt) + .param("uuid", current.getUuid()) + .execute(); + if (updated == 0) { + return null; + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logStatusChange(current, result, statusJson); + return result; + } + + public PhysicalServerResourceAssignmentVO updateFailureStatus( + PhysicalServerResourceAssignmentVO current, + String ownerState, + String providerType, + String backend, + String reasonCode, + Integer sourceRefCount, + boolean ownerObserved) { + PhysicalServerCpuAssignmentStatus status = current.getStatus() == null + ? new PhysicalServerCpuAssignmentStatus() + : JSONObjectUtil.toObject(current.getStatus(), PhysicalServerCpuAssignmentStatus.class); + status.setOwnerState(ownerState); + status.setProviderType(providerType); + status.setBackend(backend); + status.setReasonCode(reasonCode); + status.setSourceRefCount(sourceRefCount); + if (ownerObserved) { + return updateStatus(current, status, new Timestamp(System.currentTimeMillis())); + } + String statusJson = JSONObjectUtil.toJsonString(status); + int updated = SQL.New( + "update PhysicalServerResourceAssignmentVO a set a.status = :status " + + "where a.uuid = :uuid and a.specGeneration = :generation") + .param("status", statusJson) + .param("uuid", current.getUuid()) + .param("generation", current.getSpecGeneration()) + .execute(); + if (updated == 0) { + return null; + } + PhysicalServerResourceAssignmentVO result = dbf.findByUuid( + current.getUuid(), PhysicalServerResourceAssignmentVO.class); + logStatusChange(current, result, statusJson); + return result; + } + + private void logStatusChange( + PhysicalServerResourceAssignmentVO current, + PhysicalServerResourceAssignmentVO result, + String statusJson) { + if (equals(current.getStatus(), statusJson)) { + return; + } + logger.info(String.format( + "physical server resource assignment observed status changed: " + + "serverUuid[%s], resourceType[%s], assignmentType[%s], " + + "specGeneration[%s], oldStatus[%s], newStatus[%s]", + current.getServerUuid(), current.getResourceType(), current.getAssignmentType(), + result.getSpecGeneration(), current.getStatus(), statusJson)); + } + + private boolean equals(Object left, Object right) { + return left == null ? right == null : left.equals(right); + } + + private PhysicalServerResourceAssignmentVO requireAssignment( + String serverUuid, PhysicalServerCpuAssignmentType assignmentType) { + PhysicalServerResourceAssignmentVO current = find(serverUuid, assignmentType); + if (current == null) { + throw new OperationFailureException(operr( + ORG_ZSTACK_CORE_10000, + "CPU assignment[%s] does not exist on physical server[uuid:%s]", + assignmentType, serverUuid + )); + } + return current; + } + + private PhysicalServerCpuAssignmentSpec updatedSpec( + PhysicalServerResourceAssignmentVO current, + APIUpdatePhysicalServerCpuAssignmentMsg msg) { + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + current.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (msg.isEnabled() != null) { + spec.setEnabled(msg.isEnabled()); + } + if (msg.getSelectionType() != null) { + PhysicalServerCpuSelectionType selectionType = + PhysicalServerCpuSelectionType.valueOf(msg.getSelectionType()); + spec.setSelection(new PhysicalServerCpuAssignmentSpec.Selection( + selectionType.name(), normalizeSelectionValue( + selectionType, msg.getSelectionValue()))); + } + spec.setSource("USER"); + spec.setCascadeReleaseRequested(false); + spec.setDesiredCpuSet(null); + spec.setPlanHash(null); + return spec; + } + + private PhysicalServerCpuAssignmentSpec defaultSpec(PhysicalServerCpuAssignmentType assignmentType) { + PhysicalServerCpuAssignmentSpec spec = new PhysicalServerCpuAssignmentSpec(); + spec.setSource("BUILT_IN_DEFAULT"); + spec.setEnabled(true); + switch (assignmentType) { + case ZBS_EXCLUSIVE: + spec.setSelection(new PhysicalServerCpuAssignmentSpec.Selection("PERCENTAGE", "50")); + break; + case MANAGEMENT_SHARED: + case COMPUTE_SHARED: + spec.setSource("AWAITING_DEFAULT_FREEZE"); + spec.setEnabled(false); + break; + default: + throw new IllegalArgumentException(String.format("unsupported assignmentType[%s]", assignmentType)); + } + return spec; + } + + private String normalizeSelectionValue(PhysicalServerCpuSelectionType selectionType, String value) { + switch (selectionType) { + case CPU_SET: + return PhysicalServerCpuSet.normalize(value); + case COUNT: + return String.valueOf(parseBoundedNumber(value, 1, Integer.MAX_VALUE, "COUNT")); + case PERCENTAGE: + return String.valueOf(parseBoundedNumber(value, 1, 99, "PERCENTAGE")); + default: + throw new IllegalArgumentException(String.format("unsupported selectionType[%s]", selectionType)); + } + } + + private int parseBoundedNumber(String value, int min, int max, String type) { + String normalized = value == null ? "" : value.trim(); + if (!normalized.matches("[0-9]+")) { + throw new IllegalArgumentException(String.format("invalid %s value[%s]", type, value)); + } + BigInteger number = new BigInteger(normalized); + if (number.compareTo(BigInteger.valueOf(min)) < 0 || number.compareTo(BigInteger.valueOf(max)) > 0) { + throw new IllegalArgumentException(String.format( + "%s value[%s] is outside range[%s, %s]", type, value, min, max)); + } + return number.intValue(); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerConstant.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerConstant.java new file mode 100644 index 00000000000..9da1e0a27cd --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerConstant.java @@ -0,0 +1,10 @@ +package org.zstack.physicalserver; + +public interface PhysicalServerConstant { + String SERVICE_ID = "physicalServer"; + String ACTION_CATEGORY = "physicalServer"; + String CONTROL_OWNER_KEY = "physical-server-control-owner-v1"; + String CPU_RESOURCE_TYPE = "CPU"; + String ERROR_CODE = "ORG_ZSTACK_PHYSICALSERVER_10000"; + long STATUS_TTL_MILLIS = 300_000L; +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentHandler.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentHandler.java new file mode 100644 index 00000000000..532f658ad69 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentHandler.java @@ -0,0 +1,28 @@ +package org.zstack.physicalserver; + +public class PhysicalServerCpuAssignmentHandler implements PhysicalServerResourceAssignmentHandler { + @Override + public String getResourceType() { + return PhysicalServerConstant.CPU_RESOURCE_TYPE; + } + + @Override + public boolean supportsAssignmentType(String assignmentType) { + for (PhysicalServerCpuAssignmentType type : PhysicalServerCpuAssignmentType.values()) { + if (type.name().equals(assignmentType)) { + return true; + } + } + return false; + } + + @Override + public Class getSpecClass() { + return PhysicalServerCpuAssignmentSpec.class; + } + + @Override + public Class getStatusClass() { + return PhysicalServerCpuAssignmentStatus.class; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentSpec.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentSpec.java new file mode 100644 index 00000000000..351373e1158 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentSpec.java @@ -0,0 +1,98 @@ +package org.zstack.physicalserver; + +public class PhysicalServerCpuAssignmentSpec { + public static final int SCHEMA_VERSION = 1; + + private int schemaVersion = SCHEMA_VERSION; + private String source; + private boolean enabled; + private Selection selection; + private String desiredCpuSet; + private String planHash; + private boolean cascadeReleaseRequested; + + public static class Selection { + private String type; + private String value; + + public Selection() { + } + + public Selection(String type, String value) { + this.type = type; + this.value = value; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + } + + public int getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(int schemaVersion) { + this.schemaVersion = schemaVersion; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public Selection getSelection() { + return selection; + } + + public void setSelection(Selection selection) { + this.selection = selection; + } + + public String getDesiredCpuSet() { + return desiredCpuSet; + } + + public void setDesiredCpuSet(String desiredCpuSet) { + this.desiredCpuSet = desiredCpuSet; + } + + public String getPlanHash() { + return planHash; + } + + public void setPlanHash(String planHash) { + this.planHash = planHash; + } + + public boolean isCascadeReleaseRequested() { + return cascadeReleaseRequested; + } + + public void setCascadeReleaseRequested(boolean cascadeReleaseRequested) { + this.cascadeReleaseRequested = cascadeReleaseRequested; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentStatus.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentStatus.java new file mode 100644 index 00000000000..10554675daf --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentStatus.java @@ -0,0 +1,171 @@ +package org.zstack.physicalserver; + +import java.util.ArrayList; +import java.util.List; + +public class PhysicalServerCpuAssignmentStatus { + public static final int SCHEMA_VERSION = 1; + + private int schemaVersion = SCHEMA_VERSION; + private String ownerState; + private String configuredCpuSet; + private String actualCpuSet; + private String appliedPlanHash; + private String ownerAssignmentUuid; + private String ownerGeneration; + private String backend; + private String providerType; + private String reasonCode; + private Integer sourceRefCount; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private Integer requestedCount; + private Integer effectiveCount; + private Boolean clamped; + private List consumerKeys = new ArrayList<>(); + private List unresolvedHandles = new ArrayList<>(); + + public int getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(int schemaVersion) { + this.schemaVersion = schemaVersion; + } + + public String getOwnerState() { + return ownerState; + } + + public void setOwnerState(String ownerState) { + this.ownerState = ownerState; + } + + public String getConfiguredCpuSet() { + return configuredCpuSet; + } + + public void setConfiguredCpuSet(String configuredCpuSet) { + this.configuredCpuSet = configuredCpuSet; + } + + public String getActualCpuSet() { + return actualCpuSet; + } + + public void setActualCpuSet(String actualCpuSet) { + this.actualCpuSet = actualCpuSet; + } + + public String getAppliedPlanHash() { + return appliedPlanHash; + } + + public void setAppliedPlanHash(String appliedPlanHash) { + this.appliedPlanHash = appliedPlanHash; + } + + public String getOwnerAssignmentUuid() { + return ownerAssignmentUuid; + } + + public void setOwnerAssignmentUuid(String ownerAssignmentUuid) { + this.ownerAssignmentUuid = ownerAssignmentUuid; + } + + public String getOwnerGeneration() { + return ownerGeneration; + } + + public void setOwnerGeneration(String ownerGeneration) { + this.ownerGeneration = ownerGeneration; + } + + public String getBackend() { + return backend; + } + + public void setBackend(String backend) { + this.backend = backend; + } + + public String getProviderType() { + return providerType; + } + + public void setProviderType(String providerType) { + this.providerType = providerType; + } + + public String getReasonCode() { + return reasonCode; + } + + public void setReasonCode(String reasonCode) { + this.reasonCode = reasonCode; + } + + public Integer getSourceRefCount() { + return sourceRefCount; + } + + public void setSourceRefCount(Integer sourceRefCount) { + this.sourceRefCount = sourceRefCount; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public Integer getRequestedCount() { + return requestedCount; + } + + public void setRequestedCount(Integer requestedCount) { + this.requestedCount = requestedCount; + } + + public Integer getEffectiveCount() { + return effectiveCount; + } + + public void setEffectiveCount(Integer effectiveCount) { + this.effectiveCount = effectiveCount; + } + + public Boolean getClamped() { + return clamped; + } + + public void setClamped(Boolean clamped) { + this.clamped = clamped; + } + + public List getConsumerKeys() { + return consumerKeys; + } + + public void setConsumerKeys(List consumerKeys) { + this.consumerKeys = consumerKeys; + } + + public List getUnresolvedHandles() { + return unresolvedHandles; + } + + public void setUnresolvedHandles(List unresolvedHandles) { + this.unresolvedHandles = unresolvedHandles; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentType.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentType.java new file mode 100644 index 00000000000..75d1acc1dda --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuAssignmentType.java @@ -0,0 +1,7 @@ +package org.zstack.physicalserver; + +public enum PhysicalServerCpuAssignmentType { + ZBS_EXCLUSIVE, + MANAGEMENT_SHARED, + COMPUTE_SHARED +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlan.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlan.java new file mode 100644 index 00000000000..22da5e6385f --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlan.java @@ -0,0 +1,49 @@ +package org.zstack.physicalserver; + +public class PhysicalServerCpuPlan { + private String desiredCpuSet; + private String planHash; + private Integer requestedCount; + private Integer effectiveCount; + private boolean clamped; + + public String getDesiredCpuSet() { + return desiredCpuSet; + } + + public void setDesiredCpuSet(String desiredCpuSet) { + this.desiredCpuSet = desiredCpuSet; + } + + public String getPlanHash() { + return planHash; + } + + public void setPlanHash(String planHash) { + this.planHash = planHash; + } + + public Integer getRequestedCount() { + return requestedCount; + } + + public void setRequestedCount(Integer requestedCount) { + this.requestedCount = requestedCount; + } + + public Integer getEffectiveCount() { + return effectiveCount; + } + + public void setEffectiveCount(Integer effectiveCount) { + this.effectiveCount = effectiveCount; + } + + public boolean isClamped() { + return clamped; + } + + public void setClamped(boolean clamped) { + this.clamped = clamped; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlanner.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlanner.java new file mode 100644 index 00000000000..fffc7fac90d --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuPlanner.java @@ -0,0 +1,199 @@ +package org.zstack.physicalserver; + +import org.apache.commons.codec.digest.DigestUtils; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +public class PhysicalServerCpuPlanner { + public PhysicalServerCpuPlan plan( + PhysicalServerCpuAssignmentType assignmentType, + PhysicalServerCpuAssignmentSpec spec, + PhysicalServerCpuTopology topology, + Collection exclusiveCpus) { + if (!spec.isEnabled()) { + return buildPlan(assignmentType, spec, topology, new TreeSet<>(), null, false); + } + if (spec.getSelection() == null) { + throw new IllegalArgumentException("CPU_SELECTION_REQUIRED: enabled assignment requires a selection"); + } + + PhysicalServerCpuSelectionType selectionType = + PhysicalServerCpuSelectionType.valueOf(spec.getSelection().getType()); + SortedSet desired; + Integer requestedCount = null; + boolean clamped = false; + if (assignmentType == PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE) { + desired = planZbs(selectionType, spec.getSelection().getValue(), topology); + } else { + Set available = topology.getOnlineCpus(); + available.removeAll(exclusiveCpus); + if (selectionType == PhysicalServerCpuSelectionType.CPU_SET) { + desired = PhysicalServerCpuSet.parse(spec.getSelection().getValue(), topology.getOnlineCpus()); + if (!available.containsAll(desired)) { + throw new IllegalArgumentException("CPU_SET_CONFLICT: shared CPU_SET overlaps the exclusive CPU set"); + } + } else if (selectionType == PhysicalServerCpuSelectionType.COUNT) { + requestedCount = parseNumber(spec.getSelection().getValue(), 1, Integer.MAX_VALUE, "COUNT"); + desired = selectSharedCpus(requestedCount, topology, available); + clamped = desired.size() < requestedCount; + } else { + throw new IllegalArgumentException(String.format( + "CPU_SELECTION_TYPE_NOT_SUPPORTED: shared assignment does not support %s", selectionType)); + } + } + return buildPlan(assignmentType, spec, topology, desired, requestedCount, clamped); + } + + private SortedSet planZbs( + PhysicalServerCpuSelectionType selectionType, + String selectionValue, + PhysicalServerCpuTopology topology) { + SortedSet desired = new TreeSet<>(); + if (selectionType == PhysicalServerCpuSelectionType.PERCENTAGE) { + int percentage = parseNumber(selectionValue, 1, 99, "PERCENTAGE"); + List groups = topology.getCoreGroups(); + int selectedCount = groups.size() * percentage / 100; + if (selectedCount < 1) { + throw new IllegalArgumentException("CPU_TOPOLOGY_TOO_SMALL: percentage selects no complete core group"); + } + for (int index = groups.size() - selectedCount; index < groups.size(); index++) { + desired.addAll(groups.get(index).getCpus()); + } + } else if (selectionType == PhysicalServerCpuSelectionType.CPU_SET) { + desired = PhysicalServerCpuSet.parse(selectionValue, topology.getOnlineCpus()); + for (PhysicalServerCpuTopology.CoreGroup group : topology.getCoreGroups()) { + boolean intersects = !disjoint(desired, group.getCpus()); + if (intersects && !desired.containsAll(group.getCpus())) { + throw new IllegalArgumentException(String.format( + "CPU_SIBLING_SPLIT: CPU_SET splits core group[%s]", + PhysicalServerCpuSet.format(group.getCpus()))); + } + } + } else { + throw new IllegalArgumentException(String.format( + "CPU_SELECTION_TYPE_NOT_SUPPORTED: ZBS assignment does not support %s", selectionType)); + } + + if (!disjoint(desired, topology.getCpuZeroGroup().getCpus())) { + throw new IllegalArgumentException("CPU_ZERO_RESERVED: CPU0 core group must remain shared"); + } + return desired; + } + + public String normalizeZbsCpuSet(String cpuSet, PhysicalServerCpuTopology topology) { + if (cpuSet == null || cpuSet.trim().isEmpty()) { + return ""; + } + SortedSet desired = PhysicalServerCpuSet.parse(cpuSet, topology.getOnlineCpus()); + for (PhysicalServerCpuTopology.CoreGroup group : topology.getCoreGroups()) { + boolean intersects = !disjoint(desired, group.getCpus()); + if (intersects && !desired.containsAll(group.getCpus())) { + throw new IllegalArgumentException(String.format( + "CPU_SIBLING_SPLIT: CPU_SET splits core group[%s]", + PhysicalServerCpuSet.format(group.getCpus()))); + } + } + if (!disjoint(desired, topology.getCpuZeroGroup().getCpus())) { + throw new IllegalArgumentException("CPU_ZERO_RESERVED: CPU0 core group must remain shared"); + } + return PhysicalServerCpuSet.format(desired); + } + + private SortedSet selectSharedCpus( + int count, PhysicalServerCpuTopology topology, Set available) { + Map> byNuma = new LinkedHashMap<>(); + for (PhysicalServerCpuTopology.CoreGroup group : topology.getCoreGroups()) { + byNuma.computeIfAbsent(group.getNumaId(), ignored -> new ArrayList<>()).add(group); + } + + List spreadGroups = new ArrayList<>(); + int groupIndex = 0; + boolean added; + do { + added = false; + for (List groups : byNuma.values()) { + if (groupIndex < groups.size()) { + spreadGroups.add(groups.get(groupIndex)); + added = true; + } + } + groupIndex++; + } while (added); + + SortedSet selected = new TreeSet<>(); + int siblingIndex = 0; + while (selected.size() < count) { + boolean found = false; + for (PhysicalServerCpuTopology.CoreGroup group : spreadGroups) { + List siblings = new ArrayList<>(group.getCpus()); + if (siblingIndex < siblings.size() && available.contains(siblings.get(siblingIndex))) { + selected.add(siblings.get(siblingIndex)); + found = true; + if (selected.size() == count) { + return selected; + } + } + } + if (!found) { + return selected; + } + siblingIndex++; + } + return selected; + } + + private PhysicalServerCpuPlan buildPlan( + PhysicalServerCpuAssignmentType assignmentType, + PhysicalServerCpuAssignmentSpec spec, + PhysicalServerCpuTopology topology, + SortedSet desired, + Integer requestedCount, + boolean clamped) { + String desiredCpuSet = PhysicalServerCpuSet.format(desired); + String hashInput = String.join("|", + String.valueOf(PhysicalServerCpuAssignmentSpec.SCHEMA_VERSION), + assignmentType.name(), + String.valueOf(spec.isEnabled()), + spec.getSelection() == null ? "" : spec.getSelection().getType(), + spec.getSelection() == null ? "" : spec.getSelection().getValue(), + topology.fingerprint(), + desiredCpuSet); + PhysicalServerCpuPlan plan = new PhysicalServerCpuPlan(); + plan.setDesiredCpuSet(desiredCpuSet); + plan.setPlanHash(DigestUtils.sha256Hex(hashInput)); + plan.setRequestedCount(requestedCount); + plan.setEffectiveCount(desired.size()); + plan.setClamped(clamped); + return plan; + } + + private int parseNumber(String value, int min, int max, String type) { + String normalized = value == null ? "" : value.trim(); + if (!normalized.matches("[0-9]+")) { + throw new IllegalArgumentException(String.format("invalid %s value[%s]", type, value)); + } + BigInteger number = new BigInteger(normalized); + if (number.compareTo(BigInteger.valueOf(min)) < 0 || number.compareTo(BigInteger.valueOf(max)) > 0) { + throw new IllegalArgumentException(String.format( + "%s value[%s] is outside range[%s, %s]", type, value, min, max)); + } + return number.intValue(); + } + + private boolean disjoint(Collection left, Collection right) { + Set copy = new HashSet<>(left); + copy.retainAll(right); + return copy.isEmpty(); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuReconciler.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuReconciler.java new file mode 100644 index 00000000000..9bd706ad93a --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuReconciler.java @@ -0,0 +1,1740 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.thread.ChainTask; +import org.zstack.core.thread.SingleFlightTask; +import org.zstack.core.thread.SyncTaskChain; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.core.thread.ThreadFacadeImpl; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerAdapter; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerState; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.utils.Utils; +import org.zstack.utils.gson.JSONObjectUtil; +import org.zstack.utils.logging.CLogger; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.zstack.core.Platform.operr; + +public class PhysicalServerCpuReconciler { + private static final String ZBS_PROVIDER_BACKEND = "ZBS_PROVIDER"; + private static final long RECONCILE_REQUEUE_DELAY_MILLIS = 100; + private static final long RELEASE_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(4); + private static final long RELEASE_GATE_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); + private static final long RELEASE_RETRY_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1); + private static final CLogger logger = Utils.getLogger(PhysicalServerCpuReconciler.class); + + private final AtomicReference> zbsRefs = + new AtomicReference<>(Collections.emptyMap()); + private final ConcurrentMap reconcileQueues = + new ConcurrentHashMap<>(); + private final ReconcileQueueState reconcileAllQueue = new ReconcileQueueState(); + private final Set activeZbsReleases = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + + @Autowired + private ThreadFacade thdf; + @Autowired + private PluginRegistry pluginRgty; + @Autowired + private PhysicalServerAssignmentRepository assignments; + @Autowired + private PhysicalServerCpuPlanner planner; + @Autowired + private PhysicalServerSharedCpuReconciler sharedReconciler; + + public void enqueueAll() { + reconcileAllQueue.dirty.set(true); + scheduleQueuedReconcileAll(); + } + + private void scheduleQueuedReconcileAll() { + if (!reconcileAllQueue.scheduled.compareAndSet(false, true)) { + return; + } + reconcileAllQueue.dirty.set(false); + thdf.singleFlightSubmit(new SingleFlightTask(null) + .setSyncSignature("reconcile-all-physical-server-cpu") + .run(completion -> { + refreshAndEnqueueAll(); + completion.success(null); + }) + .done(result -> { + if (!result.isSuccess()) { + logger.warn(String.format( + "failed to refresh all physical server CPU assignments: %s", + result.getErrorCode())); + } + reconcileAllQueue.scheduled.set(false); + if (reconcileAllQueue.dirty.get()) { + thdf.submitTimeoutTask( + this::scheduleQueuedReconcileAll, + TimeUnit.MILLISECONDS, + RECONCILE_REQUEUE_DELAY_MILLIS); + } + })); + } + + private void refreshAndEnqueueAll() { + refreshRefs(Collections.emptySet()); + clearZbsReleaseGatesWithoutRelation(); + Set serverUuids = new HashSet<>(zbsRefs.get().keySet()); + Set zbsDefaultServers = new HashSet<>(); + for (Map.Entry entry : zbsRefs.get().entrySet()) { + ProviderResolution provider = resolveProvider(entry.getValue()); + if (entry.getValue().getReasonCode() == null && provider.provider != null) { + zbsDefaultServers.add(entry.getKey()); + } + } + zbsDefaultServers.removeAll(activeZbsReleases); + assignments.ensureDefaults( + zbsDefaultServers, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + Map adapters = consumerAdapters(); + for (PhysicalServerCpuConsumerAdapter adapter : adapters.values()) { + adapter.refreshAssociations(); + Set associated = adapter.getAssociatedServerUuids(); + serverUuids.addAll(associated); + assignments.ensureDefaults( + associated, + PhysicalServerCpuAssignmentType.valueOf(adapter.getAssignmentType())); + } + Map> assignmentSnapshots = + assignmentsByServer(assignments.listCpuAssignments()); + serverUuids.addAll(assignmentSnapshots.keySet()); + for (String serverUuid : serverUuids) { + enqueue(serverUuid, false, assignmentSnapshots.get(serverUuid)); + } + } + + public void enqueue(String serverUuid, boolean refreshFacts) { + enqueue(serverUuid, refreshFacts, null); + } + + private void enqueue( + String serverUuid, + boolean refreshFacts, + Map assignmentSnapshot) { + if (refreshFacts) { + refreshRefs(Collections.singleton(serverUuid)); + clearZbsReleaseGateWithoutRelation(serverUuid); + ensureZbsAssignment(serverUuid); + ensureSharedAssignments(serverUuid); + } + ReconcileQueueState queue = reconcileQueues.computeIfAbsent( + serverUuid, ignored -> new ReconcileQueueState()); + queue.assignmentSnapshot.set(assignmentSnapshot); + queue.dirty.set(true); + scheduleQueuedReconcile(serverUuid, queue); + } + + public void validateUpdate( + APIUpdatePhysicalServerCpuAssignmentMsg msg, Completion completion) { + PhysicalServerCpuAssignmentType assignmentType = + PhysicalServerCpuAssignmentType.valueOf(msg.getAssignmentType()); + PhysicalServerCpuAssignmentSpec spec = assignments.previewCpuUpdate(msg); + if (!spec.isEnabled()) { + if (msg.getSelectionType() == null) { + completion.success(); + return; + } + spec.setEnabled(true); + } + + collectUpdateTopology( + msg.getServerUuid(), assignmentType, + new ReturnValueCompletion(completion) { + @Override + public void success(PhysicalServerCpuTopology topology) { + validateUpdateWithTopology( + msg, assignmentType, spec, topology, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void validateUpdateWithTopology( + APIUpdatePhysicalServerCpuAssignmentMsg msg, + PhysicalServerCpuAssignmentType assignmentType, + PhysicalServerCpuAssignmentSpec spec, + PhysicalServerCpuTopology topology, + Completion completion) { + try { + Set exclusiveCpus = assignmentType + == PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE + ? Collections.emptySet() + : conservativeExclusiveCpus(msg.getServerUuid(), topology); + PhysicalServerCpuPlan plan = planner.plan( + assignmentType, spec, topology, exclusiveCpus); + if (assignmentType == PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE) { + validateZbsAgainstExplicitSharedCpuSets( + msg.getServerUuid(), plan, topology); + } + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, "%s", error.getMessage())); + return; + } + completion.success(); + } + + private void scheduleQueuedReconcile(String serverUuid, ReconcileQueueState queue) { + if (!queue.scheduled.compareAndSet(false, true)) { + return; + } + queue.dirty.set(false); + Map assignmentSnapshot = + queue.assignmentSnapshot.getAndSet(null); + submitReconcile(serverUuid, assignmentSnapshot, new Completion(null) { + @Override + public void success() { + completeQueuedReconcile(serverUuid, queue); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to reconcile CPU assignment for physical server[uuid:%s]: %s", + serverUuid, errorCode)); + completeQueuedReconcile(serverUuid, queue); + } + }); + } + + private void completeQueuedReconcile( + String serverUuid, ReconcileQueueState queue) { + queue.scheduled.set(false); + if (queue.dirty.get()) { + thdf.submitTimeoutTask( + () -> scheduleQueuedReconcile(serverUuid, queue), + TimeUnit.MILLISECONDS, + RECONCILE_REQUEUE_DELAY_MILLIS); + } + } + + private void collectUpdateTopology( + String serverUuid, + PhysicalServerCpuAssignmentType assignmentType, + ReturnValueCompletion completion) { + PhysicalServerCpuConsumerAdapter adapter = assignmentType + == PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE + ? consumerAdapters().get(PhysicalServerCpuAssignmentType.COMPUTE_SHARED.name()) + : consumerAdapters().get(assignmentType.name()); + if (adapter == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_CONSUMER_ADAPTER_MISSING: no adapter for assignmentType[%s]", + assignmentType)); + return; + } + PhysicalServerCpuConsumerState state = adapter.getState(serverUuid); + if (state != PhysicalServerCpuConsumerState.AVAILABLE) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "CPU_CONSUMER_%s: physical server[uuid:%s], assignmentType[%s]", + state, serverUuid, assignmentType)); + return; + } + adapter.collectTopology(serverUuid, completion); + } + + private Set conservativeExclusiveCpus( + String serverUuid, PhysicalServerCpuTopology topology) { + PhysicalServerResourceAssignmentVO zbs = assignments.find( + serverUuid, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + if (zbs == null) { + return Collections.emptySet(); + } + PhysicalServerCpuAssignmentSpec zbsSpec = JSONObjectUtil.toObject( + zbs.getSpec(), PhysicalServerCpuAssignmentSpec.class); + String actual = null; + if (zbs.getStatus() != null) { + actual = JSONObjectUtil.toObject( + zbs.getStatus(), PhysicalServerCpuAssignmentStatus.class) + .getActualCpuSet(); + } + String conservative = PhysicalServerCpuSet.union( + zbsSpec.getDesiredCpuSet(), actual); + if (zbsSpec.isEnabled() && conservative.isEmpty()) { + throw new IllegalArgumentException( + "ZBS_CPUSET_NOT_READY: cannot validate SHARED before the ZBS CPUSet is known"); + } + return conservative.isEmpty() + ? Collections.emptySet() + : PhysicalServerCpuSet.parse( + conservative, topology.getOnlineCpus()); + } + + private void validateZbsAgainstExplicitSharedCpuSets( + String serverUuid, + PhysicalServerCpuPlan zbsPlan, + PhysicalServerCpuTopology topology) { + Set zbsCpus = PhysicalServerCpuSet.parse( + zbsPlan.getDesiredCpuSet(), topology.getOnlineCpus()); + for (PhysicalServerCpuAssignmentType type : new PhysicalServerCpuAssignmentType[]{ + PhysicalServerCpuAssignmentType.COMPUTE_SHARED, + PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED}) { + PhysicalServerResourceAssignmentVO shared = assignments.find(serverUuid, type); + if (shared == null) { + continue; + } + PhysicalServerCpuAssignmentSpec sharedSpec = JSONObjectUtil.toObject( + shared.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (!sharedSpec.isEnabled() + || sharedSpec.getSelection() == null + || !PhysicalServerCpuSelectionType.CPU_SET.name().equals( + sharedSpec.getSelection().getType())) { + continue; + } + Set sharedCpus = PhysicalServerCpuSet.parse( + sharedSpec.getSelection().getValue(), topology.getOnlineCpus()); + sharedCpus.retainAll(zbsCpus); + if (!sharedCpus.isEmpty()) { + throw new IllegalArgumentException(String.format( + "CPU_SET_CONFLICT: ZBS CPU_SET overlaps %s CPU_SET[%s]", + type, PhysicalServerCpuSet.format(sharedCpus))); + } + } + } + + public void releaseZbsAssignment(String serverUuid, Completion completion) { + if (!activeZbsReleases.add(serverUuid)) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS_CPU_ISOLATION_RELEASE_IN_PROGRESS: physical server[uuid:%s] is already releasing", + serverUuid)); + return; + } + long deadline = System.currentTimeMillis() + RELEASE_TIMEOUT_MILLIS; + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(serverUuid); + } + + @Override + public void run(SyncTaskChain chain) { + runZbsRelease(serverUuid, deadline, chain, completion); + } + + @Override + public String getName() { + return String.format("release-zbs-cpu-isolation-%s", serverUuid); + } + }); + } + + public void releaseAssignment( + String serverUuid, + PhysicalServerCpuAssignmentType assignmentType, + String consumerUuid, + Completion completion) { + if (assignmentType == PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE) { + releaseZbsAssignment(serverUuid, completion); + return; + } + ReconcileQueueState queue = reconcileQueues.computeIfAbsent( + serverUuid, ignored -> new ReconcileQueueState()); + if (!queue.activeSharedReleases.add(assignmentType)) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "SHARED_CPUSET_RELEASE_IN_PROGRESS: physical server[uuid:%s], assignmentType[%s]", + serverUuid, assignmentType)); + return; + } + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(serverUuid); + } + + @Override + public void run(SyncTaskChain chain) { + sharedReconciler.release( + serverUuid, + assignmentType, + consumerUuid, + new Completion(completion) { + @Override + public void success() { + scheduleSharedReleaseGateExpiry( + serverUuid, assignmentType, queue); + try { + completion.success(); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + queue.activeSharedReleases.remove(assignmentType); + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + @Override + public String getName() { + return String.format( + "release-shared-cpuset-%s-%s", serverUuid, assignmentType); + } + }); + } + + private void runZbsRelease( + String serverUuid, + long deadline, + SyncTaskChain chain, + Completion completion) { + AtomicReference prepared = + new AtomicReference<>(); + AtomicBoolean completed = new AtomicBoolean(); + Completion tracked = new Completion(completion) { + @Override + public void success() { + if (!completed.compareAndSet(false, true)) { + return; + } + scheduleZbsReleaseGateExpiry(serverUuid); + try { + completion.success(); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + if (!completed.compareAndSet(false, true)) { + return; + } + activeZbsReleases.remove(serverUuid); + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }; + try { + failIfReleaseExpired(serverUuid, deadline); + refreshRefs(Collections.singleton(serverUuid)); + failIfReleaseExpired(serverUuid, deadline); + PhysicalServerResourceAssignmentVO assignment = + assignments.prepareZbsRelease(serverUuid); + prepared.set(assignment); + if (assignment == null) { + activeZbsReleases.remove(serverUuid); + tracked.success(); + return; + } + reconcileZbsRelease( + serverUuid, + assignment, + deadline, + new Completion(tracked) { + @Override + public void success() { + tracked.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + failZbsReleaseAndRestore( + serverUuid, errorCode, tracked); + } + }); + } catch (OperationFailureException error) { + if (prepared.get() == null) { + tracked.fail(error.getErrorCode()); + } else { + failZbsReleaseAndRestore( + serverUuid, error.getErrorCode(), tracked); + } + } catch (RuntimeException error) { + ErrorCode errorCode = operr( + PhysicalServerConstant.ERROR_CODE, + "failed to prepare ZBS CPU isolation release on physical server[uuid:%s]: %s", + serverUuid, error.getMessage()); + if (prepared.get() == null) { + tracked.fail(errorCode); + } else { + failZbsReleaseAndRestore(serverUuid, errorCode, tracked); + } + } + } + + private void failZbsReleaseAndRestore( + String serverUuid, + ErrorCode releaseError, + Completion completion) { + restoreAfterFailedRelease( + serverUuid, + new Completion(completion) { + @Override + public void success() { + completion.fail(releaseError); + } + + @Override + public void fail(ErrorCode restoreError) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "failed to release ZBS CPU isolation on physical server[uuid:%s]: %s; " + + "failed to restore the assignment: %s", + serverUuid, releaseError, restoreError)); + } + }); + } + + private void restoreAfterFailedRelease( + String serverUuid, Completion completion) { + try { + PhysicalServerResourceAssignmentVO restored = + assignments.cancelZbsRelease(serverUuid); + if (restored != null) { + enqueue(serverUuid, true); + } + completion.success(); + } catch (OperationFailureException error) { + completion.fail(error.getErrorCode()); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "failed to restore ZBS CPU isolation assignment on physical server[uuid:%s]: %s", + serverUuid, error.getMessage())); + } + } + + private void submitReconcile( + String serverUuid, + Map assignmentSnapshot, + Completion completion) { + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + return serverOperationSignature(serverUuid); + } + + @Override + public void run(SyncTaskChain chain) { + reconcileLatest( + serverUuid, + assignmentSnapshot, + new ReturnValueCompletion(completion) { + @Override + public void success(Object returnValue) { + try { + completion.success(); + } finally { + chain.next(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + try { + completion.fail(errorCode); + } finally { + chain.next(); + } + } + }); + } + + @Override + public String getName() { + return String.format("reconcile-physical-server-cpu-%s", serverUuid); + } + }); + } + + private String serverOperationSignature(String serverUuid) { + return String.format("physical-server-cpu-operation-%s", serverUuid); + } + + private void reconcileZbsRelease( + String serverUuid, + PhysicalServerResourceAssignmentVO assignment, + long deadline, + Completion completion) { + if (!releaseAssignmentStillCurrent(assignment)) { + completion.fail(releaseSuperseded(serverUuid)); + return; + } + if (System.currentTimeMillis() >= deadline) { + completion.fail(releaseTimeout(serverUuid)); + return; + } + long timeout = remainingReleaseBudget(deadline); + if (timeout <= 0) { + completion.fail(releaseTimeout(serverUuid)); + return; + } + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + ProviderResolution resolution = resolveProvider(ref); + if (ref == null || ref.getReasonCode() != null || resolution.provider == null) { + String reason = ref == null + ? "ZBS_NODE_RELATION_MISSING" + : ref.getReasonCode() != null + ? ref.getReasonCode() : resolution.reasonCode; + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "cannot release ZBS CPU isolation on physical server[uuid:%s], reason[%s]", + serverUuid, reason)); + return; + } + queryProvider( + resolution.provider, + ref, + timeout, + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + handleZbsReleaseFact( + assignment, + ref, + resolution.provider, + fact, + deadline, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void handleZbsReleaseFact( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + ZbsCpuIsolationFact fact, + long deadline, + Completion completion) { + String serverUuid = assignment.getServerUuid(); + if (System.currentTimeMillis() >= deadline) { + completion.fail(releaseTimeout(serverUuid)); + return; + } + String invalidReason = validateZbsReleaseFact(fact); + if (invalidReason != null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "cannot release ZBS CPU isolation on physical server[uuid:%s], reason[%s]", + serverUuid, invalidReason)); + return; + } + if ("DISABLED".equals(fact.getIsolationState()) + && empty(fact.getEffectiveCpuSet()) + && ownerVersionMatches(assignment, fact)) { + completeZbsRelease(assignment, deadline, completion); + return; + } + if (!releaseAssignmentStillCurrent(assignment)) { + completion.fail(releaseSuperseded(serverUuid)); + return; + } + long timeout = remainingReleaseBudget(deadline); + if (timeout <= 0) { + completion.fail(releaseTimeout(serverUuid)); + return; + } + + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + ZbsCpuIsolationUpdate update = new ZbsCpuIsolationUpdate(); + update.setAssignmentUuid(assignment.getUuid()); + update.setOperation("DISABLE"); + update.setPlanHash(spec.getPlanHash()); + update.setRevision(assignment.getSpecGeneration()); + updateProvider( + provider, + ref, + update, + timeout, + new Completion(completion) { + @Override + public void success() { + scheduleZbsReleasePoll(assignment, deadline, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private String validateZbsReleaseFact(ZbsCpuIsolationFact fact) { + if (fact == null) { + return "OWNER_FACT_INVALID"; + } + if (!"1".equals(fact.getCapabilityVersion())) { + return "PROVIDER_VERSION_UNSUPPORTED"; + } + if (!isOwnerState(fact.getIsolationState()) || fact.getObservedAt() == null) { + return "OWNER_FACT_INVALID"; + } + if ("UNSUPPORTED".equals(fact.getIsolationState())) { + return fact.getReasonCode() == null + ? "OWNER_UPDATE_UNSUPPORTED" : fact.getReasonCode(); + } + return null; + } + + private void completeZbsRelease( + PhysicalServerResourceAssignmentVO assignment, + long deadline, + Completion completion) { + String serverUuid = assignment.getServerUuid(); + if (System.currentTimeMillis() >= deadline) { + completion.fail(releaseTimeout(serverUuid)); + return; + } + if (!releaseAssignmentStillCurrent(assignment)) { + completion.fail(releaseSuperseded(serverUuid)); + return; + } + if (!assignments.deleteIfGeneration( + assignment.getUuid(), assignment.getSpecGeneration())) { + completion.fail(releaseSuperseded(serverUuid)); + return; + } + refreshHostCapacity(serverUuid); + completion.success(); + } + + private void scheduleZbsReleasePoll( + PhysicalServerResourceAssignmentVO assignment, + long deadline, + Completion completion) { + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + completion.fail(releaseTimeout(assignment.getServerUuid())); + return; + } + thdf.submitTimeoutTask( + () -> reconcileZbsRelease( + assignment.getServerUuid(), assignment, deadline, completion), + TimeUnit.MILLISECONDS, + Math.min(RELEASE_RETRY_INTERVAL_MILLIS, remaining)); + } + + private boolean releaseAssignmentStillCurrent( + PhysicalServerResourceAssignmentVO expected) { + PhysicalServerResourceAssignmentVO current = assignments.find( + expected.getServerUuid(), PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + if (current == null + || !current.getUuid().equals(expected.getUuid()) + || current.getSpecGeneration() != expected.getSpecGeneration()) { + return false; + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + current.getSpec(), PhysicalServerCpuAssignmentSpec.class); + return spec.isCascadeReleaseRequested(); + } + + private long remainingReleaseBudget(long deadline) { + long remaining = deadline - System.currentTimeMillis(); + return Math.min( + remaining, + TimeUnit.SECONDS.toMillis( + PhysicalServerGlobalConfig.ZBS_PROVIDER_CALL_TIMEOUT.value(Long.class))); + } + + private void failIfReleaseExpired(String serverUuid, long deadline) { + if (System.currentTimeMillis() >= deadline) { + throw new OperationFailureException(releaseTimeout(serverUuid)); + } + } + + private ErrorCode releaseTimeout(String serverUuid) { + return operr( + PhysicalServerConstant.ERROR_CODE, + "timed out waiting for ZBS to report disabled CPU isolation on physical server[uuid:%s]", + serverUuid); + } + + private ErrorCode releaseSuperseded(String serverUuid) { + return operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS CPU isolation release on physical server[uuid:%s] was superseded by a concurrent assignment update", + serverUuid); + } + + private boolean empty(String value) { + return value == null || value.trim().isEmpty(); + } + + private void scheduleSharedReleaseGateExpiry( + String serverUuid, + PhysicalServerCpuAssignmentType assignmentType, + ReconcileQueueState queue) { + thdf.submitTimeoutTask(() -> { + if (!queue.activeSharedReleases.remove(assignmentType)) { + return; + } + logger.warn(String.format( + "shared CPUSet release gate expired before the consumer relation disappeared: " + + "serverUuid[%s], assignmentType[%s]", + serverUuid, assignmentType)); + enqueue(serverUuid, false); + }, TimeUnit.MILLISECONDS, RELEASE_GATE_TIMEOUT_MILLIS); + } + + private boolean keepSharedReleaseGate(String serverUuid) { + ReconcileQueueState queue = reconcileQueues.get(serverUuid); + PhysicalServerCpuAssignmentType type = + PhysicalServerCpuAssignmentType.COMPUTE_SHARED; + if (queue == null || !queue.activeSharedReleases.contains(type)) { + return false; + } + PhysicalServerCpuConsumerAdapter adapter = + consumerAdapters().get(type.name()); + if (adapter == null + || adapter.getState(serverUuid) != PhysicalServerCpuConsumerState.MISSING) { + return true; + } + queue.activeSharedReleases.remove(type); + return false; + } + + private void scheduleZbsReleaseGateExpiry(String serverUuid) { + thdf.submitTimeoutTask(() -> { + if (!activeZbsReleases.remove(serverUuid)) { + return; + } + logger.warn(String.format( + "ZBS CPU isolation release gate expired before the storage relation disappeared: " + + "serverUuid[%s]", + serverUuid)); + enqueue(serverUuid, true); + }, TimeUnit.MILLISECONDS, RELEASE_GATE_TIMEOUT_MILLIS); + } + + private void clearZbsReleaseGatesWithoutRelation() { + for (String serverUuid : new HashSet<>(activeZbsReleases)) { + clearZbsReleaseGateWithoutRelation(serverUuid); + } + } + + private void clearZbsReleaseGateWithoutRelation(String serverUuid) { + if (!zbsRefs.get().containsKey(serverUuid)) { + activeZbsReleases.remove(serverUuid); + } + } + + private void refreshRefs(Collection serverUuids) { + List contributors = + pluginRgty.getExtensionList(ZbsNodeRefContributor.class); + Map loaded = new HashMap<>(); + for (ZbsNodeRefContributor contributor : contributors) { + Map contribution; + try { + contribution = contributor.bulkList(serverUuids); + } catch (RuntimeException error) { + logger.warn(String.format( + "failed to refresh ZBS node relations from contributor[%s]; " + + "keeping the last observed relation snapshot: %s", + contributor.getClass().getName(), error.getMessage())); + return; + } + for (Map.Entry entry : contribution.entrySet()) { + if (loaded.put(entry.getKey(), entry.getValue()) != null) { + entry.getValue().setReasonCode("ZBS_NODE_REF_CONTRIBUTOR_AMBIGUOUS"); + } + } + } + + if (serverUuids == null || serverUuids.isEmpty()) { + zbsRefs.set(Collections.unmodifiableMap(loaded)); + return; + } + + while (true) { + Map current = zbsRefs.get(); + Map replacement = new HashMap<>(current); + for (String serverUuid : serverUuids) { + replacement.remove(serverUuid); + } + replacement.putAll(loaded); + if (zbsRefs.compareAndSet(current, Collections.unmodifiableMap(replacement))) { + return; + } + } + } + + private void ensureZbsAssignment(String serverUuid) { + if (activeZbsReleases.contains(serverUuid)) { + return; + } + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + ProviderResolution provider = resolveProvider(ref); + if (ref != null && ref.getReasonCode() == null && provider.provider != null) { + assignments.ensureDefault( + serverUuid, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + } + } + + private void ensureSharedAssignments(String serverUuid) { + for (PhysicalServerCpuConsumerAdapter adapter : consumerAdapters().values()) { + if (adapter.getState(serverUuid) != PhysicalServerCpuConsumerState.MISSING) { + assignments.ensureDefault( + serverUuid, + PhysicalServerCpuAssignmentType.valueOf(adapter.getAssignmentType())); + } + } + } + + private void reconcileLatest( + String serverUuid, + Map assignmentSnapshot, + ReturnValueCompletion completion) { + reconcileZbsLatest(serverUuid, assignmentSnapshot, + new ReturnValueCompletion(completion) { + @Override + public void success(Object zbsAssignment) { + reconcileShared( + serverUuid, + assignmentSnapshot, + (PhysicalServerResourceAssignmentVO) zbsAssignment, + null, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to reconcile ZBS CPU isolation for physical server[uuid:%s]: %s", + serverUuid, errorCode)); + reconcileShared( + serverUuid, + assignmentSnapshot, + assignmentSnapshot == null ? null : assignmentSnapshot.get( + PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE), + errorCode, + completion); + } + }); + } + + private void reconcileShared( + String serverUuid, + Map assignmentSnapshot, + PhysicalServerResourceAssignmentVO zbsAssignment, + ErrorCode zbsError, + ReturnValueCompletion completion) { + boolean skipCompute = keepSharedReleaseGate(serverUuid); + sharedReconciler.reconcile( + serverUuid, + assignmentSnapshot, + zbsAssignment, + skipCompute, + new Completion(completion) { + @Override + public void success() { + completeAfterShared(zbsError, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to reconcile shared CPUSet for physical server[uuid:%s]: %s", + serverUuid, errorCode)); + completeAfterShared(zbsError, completion); + } + }); + } + + private void completeAfterShared( + ErrorCode zbsError, ReturnValueCompletion completion) { + if (zbsError == null) { + completion.success(null); + } else { + completion.fail(zbsError); + } + } + + private void reconcileZbsLatest( + String serverUuid, ReturnValueCompletion completion) { + reconcileZbsLatest(serverUuid, null, completion); + } + + private void reconcileZbsLatest( + String serverUuid, + Map assignmentSnapshot, + ReturnValueCompletion completion) { + PhysicalServerResourceAssignmentVO assignment = assignmentSnapshot == null + ? assignments.find( + serverUuid, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE) + : assignmentSnapshot.get(PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE); + if (assignment == null) { + completion.success(null); + return; + } + + PhysicalServerCpuAssignmentSpec currentSpec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (currentSpec.isCascadeReleaseRequested() + && !activeZbsReleases.contains(serverUuid)) { + assignments.cancelZbsRelease(serverUuid); + reconcileZbsLatest(serverUuid, completion); + return; + } + + ZbsNodeRef ref = zbsRefs.get().get(serverUuid); + if (ref == null) { + persistFailure(assignment, null, "ZBS_NODE_RELATION_MISSING", null, false, completion); + return; + } + if (ref.getReasonCode() != null) { + persistFailure(assignment, null, ref.getReasonCode(), ref.getSourceRefCount(), true, completion); + return; + } + + ProviderResolution resolution = resolveProvider(ref); + if (resolution.provider == null) { + persistFailure(assignment, resolution.providerType, resolution.reasonCode, + ref.getSourceRefCount(), true, completion); + return; + } + + PhysicalServerCpuConsumerAdapter adapter = consumerAdapters().get( + PhysicalServerCpuAssignmentType.COMPUTE_SHARED.name()); + PhysicalServerCpuConsumerState state = adapter == null + ? PhysicalServerCpuConsumerState.MISSING : adapter.getState(serverUuid); + if (state == PhysicalServerCpuConsumerState.MISSING) { + persistFailure(assignment, resolution.provider.getProviderType(), "CPU_TOPOLOGY_SOURCE_MISSING", + ref.getSourceRefCount(), true, completion); + return; + } + if (state == PhysicalServerCpuConsumerState.UNAVAILABLE) { + persistFailure(assignment, resolution.provider.getProviderType(), + "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE", + ref.getSourceRefCount(), false, completion); + return; + } + adapter.collectTopology(serverUuid, + new ReturnValueCompletion(completion) { + @Override + public void success(PhysicalServerCpuTopology topology) { + reconcileWithTopology( + assignment, + ref, + resolution.provider, + topology, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to query CPU topology for physical server[uuid:%s]: %s", + serverUuid, errorCode)); + persistFailure(assignment, resolution.provider.getProviderType(), + "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE", + ref.getSourceRefCount(), false, completion); + } + }); + } + + private void reconcileWithTopology( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ReturnValueCompletion completion) { + planZbsAssignment( + assignment, + topology, + new ReturnValueCompletion(completion) { + @Override + public void success(PhysicalServerCpuPlan plan) { + persistPlanAndQuery( + assignment, ref, provider, topology, plan, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + assignment, + provider.getProviderType(), + PhysicalServerReasonCodes.from(errorCode), + ref.getSourceRefCount(), + true, + completion); + } + }); + } + + private void planZbsAssignment( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuTopology topology, + ReturnValueCompletion completion) { + PhysicalServerCpuPlan plan; + try { + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + plan = planner.plan( + PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE, + spec, + topology, + Collections.emptySet()); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, "%s", error.getMessage())); + return; + } + completion.success(plan); + } + + private void persistPlanAndQuery( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + PhysicalServerCpuPlan plan, + ReturnValueCompletion completion) { + PhysicalServerResourceAssignmentVO planned = assignments.updatePlan(assignment, plan); + if (planned == null || planned.getSpecGeneration() != assignment.getSpecGeneration()) { + reconcileZbsLatest(assignment.getServerUuid(), completion); + return; + } + queryAndConverge(planned, ref, provider, topology, completion); + } + + private void queryAndConverge( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ReturnValueCompletion completion) { + queryProvider(provider, ref, new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + validateFactAndConverge( + assignment, ref, provider, topology, fact, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure(assignment, provider.getProviderType(), + "OWNER_UNREACHABLE", ref.getSourceRefCount(), false, completion); + } + }); + } + + private void validateFactAndConverge( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ZbsCpuIsolationFact fact, + ReturnValueCompletion completion) { + validateFact( + assignment, + ref, + provider, + topology, + fact, + null, + new ReturnValueCompletion(completion) { + @Override + public void success(ValidatedFact validated) { + convergeValidatedFact( + assignment, + ref, + provider, + topology, + fact, + validated, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + assignment, + provider.getProviderType(), + PhysicalServerReasonCodes.from(errorCode), + ref.getSourceRefCount(), + true, + completion); + } + }); + } + + private void convergeValidatedFact( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ZbsCpuIsolationFact fact, + ValidatedFact validated, + ReturnValueCompletion completion) { + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + if (!shouldUpdate(assignment, spec, fact, validated.status)) { + recordStatus(assignment, validated, completion); + return; + } + + ZbsCpuIsolationUpdate update = new ZbsCpuIsolationUpdate(); + update.setAssignmentUuid(assignment.getUuid()); + boolean wantsEnabled = spec.isEnabled() + && !spec.isCascadeReleaseRequested(); + update.setOperation(wantsEnabled ? "SET" : "DISABLE"); + update.setDesiredCpuSet(wantsEnabled ? spec.getDesiredCpuSet() : null); + update.setPlanHash(spec.getPlanHash()); + update.setRevision(assignment.getSpecGeneration()); + logger.info(String.format( + "requesting ZBS CPU isolation update: serverUuid[%s], providerType[%s], " + + "assignmentUuid[%s], operation[%s], desiredCpuSet[%s], planHash[%s], generation[%s]", + assignment.getServerUuid(), provider.getProviderType(), + assignment.getUuid(), update.getOperation(), update.getDesiredCpuSet(), update.getPlanHash(), + update.getRevision())); + updateProvider(provider, ref, update, new Completion(completion) { + @Override + public void success() { + queryAfterUpdate(assignment, ref, provider, topology, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + if (isProviderCallUnavailable(errorCode)) { + persistFailure(assignment, provider.getProviderType(), + "OWNER_UNREACHABLE", ref.getSourceRefCount(), false, completion); + return; + } + recordFact(assignment, ref, provider, topology, + fact, "OWNER_UPDATE_REJECTED", completion); + } + }); + } + + private void queryAfterUpdate( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ReturnValueCompletion completion) { + queryProvider(provider, ref, new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + recordFact( + assignment, + ref, + provider, + topology, + fact, + ownerVersionMatches(assignment, fact) + ? null : "OWNER_GENERATION_STALE", + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure(assignment, provider.getProviderType(), + "OWNER_UNREACHABLE", ref.getSourceRefCount(), false, completion); + } + }); + } + + private boolean shouldUpdate( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuAssignmentSpec spec, + ZbsCpuIsolationFact fact, + PhysicalServerCpuAssignmentStatus status) { + if (!("USER".equals(spec.getSource()) || spec.isCascadeReleaseRequested()) + || fact == null) { + return false; + } + if ("UNSUPPORTED".equals(fact.getIsolationState())) { + return false; + } + if (!ownerVersionMatches(assignment, fact)) { + return true; + } + if (!spec.isEnabled() || spec.isCascadeReleaseRequested()) { + return !"DISABLED".equals(fact.getIsolationState()); + } + if (!"READY".equals(fact.getIsolationState())) { + return true; + } + return !spec.getDesiredCpuSet().equals(status.getActualCpuSet()); + } + + private boolean ownerVersionMatches( + PhysicalServerResourceAssignmentVO assignment, + ZbsCpuIsolationFact fact) { + return fact != null + && assignment.getUuid().equals(fact.getOwnerAssignmentUuid()) + && String.valueOf(assignment.getSpecGeneration()).equals( + fact.getOwnerGeneration()); + } + + private void queryProvider( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ReturnValueCompletion completion) { + queryProvider( + provider, + ref, + TimeUnit.SECONDS.toMillis( + PhysicalServerGlobalConfig.ZBS_PROVIDER_CALL_TIMEOUT.value(Long.class)), + completion); + } + + private void queryProvider( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + long timeoutMillis, + ReturnValueCompletion completion) { + AtomicBoolean completed = new AtomicBoolean(); + long boundedTimeout = Math.max(1, timeoutMillis); + ThreadFacadeImpl.TimeoutTaskReceipt receipt = thdf.submitTimeoutTask(() -> { + if (completed.compareAndSet(false, true)) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS_PROVIDER_CALL_TIMEOUT: Provider[%s] Query timed out after %s milliseconds", + provider.getProviderType(), boundedTimeout)); + } + }, TimeUnit.MILLISECONDS, boundedTimeout); + ReturnValueCompletion guarded = + new ReturnValueCompletion(completion) { + @Override + public void success(ZbsCpuIsolationFact fact) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.success(fact); + } + } + + @Override + public void fail(ErrorCode errorCode) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.fail(errorCode); + } + } + }; + invokeProviderQuery(provider, ref, guarded); + } + + private void invokeProviderQuery( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ReturnValueCompletion completion) { + try { + provider.query(ref, completion); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS_PROVIDER_CALL_FAILED: Provider[%s] Query failed: %s", + provider.getProviderType(), error.getMessage())); + } + } + + private void updateProvider( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ZbsCpuIsolationUpdate update, + Completion completion) { + updateProvider( + provider, + ref, + update, + TimeUnit.SECONDS.toMillis( + PhysicalServerGlobalConfig.ZBS_PROVIDER_CALL_TIMEOUT.value(Long.class)), + completion); + } + + private void updateProvider( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ZbsCpuIsolationUpdate update, + long timeoutMillis, + Completion completion) { + AtomicBoolean completed = new AtomicBoolean(); + long boundedTimeout = Math.max(1, timeoutMillis); + ThreadFacadeImpl.TimeoutTaskReceipt receipt = thdf.submitTimeoutTask(() -> { + if (completed.compareAndSet(false, true)) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS_PROVIDER_CALL_TIMEOUT: Provider[%s] Update timed out after %s milliseconds", + provider.getProviderType(), boundedTimeout)); + } + }, TimeUnit.MILLISECONDS, boundedTimeout); + Completion guarded = new Completion(completion) { + @Override + public void success() { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.success(); + } + } + + @Override + public void fail(ErrorCode errorCode) { + if (completed.compareAndSet(false, true)) { + receipt.cancel(); + completion.fail(errorCode); + } + } + }; + invokeProviderUpdate(provider, ref, update, guarded); + } + + private void invokeProviderUpdate( + ZbsCpuIsolationProvider provider, + ZbsNodeRef ref, + ZbsCpuIsolationUpdate update, + Completion completion) { + try { + provider.update(ref, update, completion); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS_PROVIDER_CALL_FAILED: Provider[%s] Update failed: %s", + provider.getProviderType(), error.getMessage())); + } + } + + private boolean isProviderCallUnavailable(ErrorCode errorCode) { + String details = errorCode == null ? null : errorCode.getDetails(); + return details != null && (details.contains("ZBS_PROVIDER_CALL_TIMEOUT") + || details.contains("ZBS_PROVIDER_CALL_FAILED") + || details.contains("unhandled exception happened when calling")); + } + + private void recordFact( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ZbsCpuIsolationFact fact, + String overrideReasonCode, + ReturnValueCompletion completion) { + validateFact( + assignment, + ref, + provider, + topology, + fact, + overrideReasonCode, + new ReturnValueCompletion(completion) { + @Override + public void success(ValidatedFact validated) { + recordStatus(assignment, validated, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + assignment, + provider.getProviderType(), + PhysicalServerReasonCodes.from(errorCode), + ref.getSourceRefCount(), + true, + completion); + } + }); + } + + private void validateFact( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ZbsCpuIsolationFact fact, + String overrideReasonCode, + ReturnValueCompletion completion) { + ValidatedFact validated; + try { + PhysicalServerCpuAssignmentStatus status = toStatus( + assignment, ref, provider, topology, fact, overrideReasonCode); + validated = new ValidatedFact(status, fact.getObservedAt()); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, "%s", error.getMessage())); + return; + } + completion.success(validated); + } + + private void recordStatus( + PhysicalServerResourceAssignmentVO assignment, + ValidatedFact validated, + ReturnValueCompletion completion) { + String previousCapacityFact = capacityFact(assignment.getStatus()); + PhysicalServerResourceAssignmentVO updated = assignments.updateStatus( + assignment, validated.status, validated.observedAt); + if (updated == null) { + reconcileZbsLatest(assignment.getServerUuid(), completion); + return; + } + if (!previousCapacityFact.equals(capacityFact(validated.status))) { + refreshHostCapacity(assignment.getServerUuid()); + } + completion.success(updated); + } + + private String capacityFact(String statusJson) { + if (statusJson == null) { + return ""; + } + return capacityFact(JSONObjectUtil.toObject( + statusJson, PhysicalServerCpuAssignmentStatus.class)); + } + + private String capacityFact(PhysicalServerCpuAssignmentStatus status) { + String ownerState = status.getOwnerState() == null ? "" : status.getOwnerState(); + String actualCpuSet = status.getActualCpuSet() == null ? "" : status.getActualCpuSet(); + return ownerState + ":" + actualCpuSet; + } + + private PhysicalServerCpuAssignmentStatus toStatus( + PhysicalServerResourceAssignmentVO assignment, + ZbsNodeRef ref, + ZbsCpuIsolationProvider provider, + PhysicalServerCpuTopology topology, + ZbsCpuIsolationFact fact, + String overrideReasonCode) { + if (fact == null) { + throw new IllegalArgumentException("OWNER_FACT_INVALID: Provider returned null"); + } + if (!"1".equals(fact.getCapabilityVersion())) { + throw new IllegalArgumentException("PROVIDER_VERSION_UNSUPPORTED: capabilityVersion is not 1"); + } + if (!isOwnerState(fact.getIsolationState())) { + throw new IllegalArgumentException("OWNER_FACT_INVALID: isolationState is not supported"); + } + if (fact.getObservedAt() == null) { + throw new IllegalArgumentException("OWNER_FACT_INVALID: observedAt is missing"); + } + + String configured = planner.normalizeZbsCpuSet(fact.getConfiguredCpuSet(), topology); + String actual = planner.normalizeZbsCpuSet(fact.getEffectiveCpuSet(), topology); + if ("READY".equals(fact.getIsolationState())) { + if (configured.isEmpty() || !configured.equals(actual)) { + throw new IllegalArgumentException( + "OWNER_FACT_INVALID: READY requires identical non-empty configured/effective CPUSet"); + } + if (!fact.isServiceReady() + || fact.getCoveredServiceCount() == null + || fact.getExpectedServiceCount() == null + || !fact.getCoveredServiceCount().equals(fact.getExpectedServiceCount())) { + throw new IllegalArgumentException( + "OWNER_SERVICE_NOT_READY: READY requires complete service coverage"); + } + } + if ("DISABLED".equals(fact.getIsolationState()) && !actual.isEmpty()) { + throw new IllegalArgumentException( + "OWNER_FACT_INVALID: DISABLED must not report an effective CPUSet"); + } + + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + String reason = overrideReasonCode != null ? overrideReasonCode : fact.getReasonCode(); + boolean wantsEnabled = spec.isEnabled() + && !spec.isCascadeReleaseRequested(); + boolean desiredMatches = wantsEnabled + ? spec.getDesiredCpuSet().equals(actual) && "READY".equals(fact.getIsolationState()) + : actual.isEmpty() && "DISABLED".equals(fact.getIsolationState()); + if (!desiredMatches && reason == null) { + reason = "ACTUAL_MISMATCH"; + } + + PhysicalServerCpuAssignmentStatus status = new PhysicalServerCpuAssignmentStatus(); + status.setOwnerState("OWNER_UPDATE_REJECTED".equals(overrideReasonCode) + ? "ERROR" + : "OWNER_GENERATION_STALE".equals(overrideReasonCode) + ? "PENDING" : fact.getIsolationState()); + status.setConfiguredCpuSet(configured); + status.setActualCpuSet(actual); + status.setAppliedPlanHash(desiredMatches ? spec.getPlanHash() : null); + status.setOwnerAssignmentUuid(fact.getOwnerAssignmentUuid()); + status.setOwnerGeneration(fact.getOwnerGeneration()); + status.setBackend(ZBS_PROVIDER_BACKEND); + status.setProviderType(provider.getProviderType()); + status.setReasonCode(reason); + status.setSourceRefCount(ref.getSourceRefCount()); + status.setCoveredServiceCount(fact.getCoveredServiceCount()); + status.setExpectedServiceCount(fact.getExpectedServiceCount()); + return status; + } + + private boolean isOwnerState(String value) { + return "READY".equals(value) + || "PENDING".equals(value) + || "UNSUPPORTED".equals(value) + || "ERROR".equals(value) + || "DISABLED".equals(value); + } + + private void persistFailure( + PhysicalServerResourceAssignmentVO assignment, + String providerType, + String reasonCode, + Integer sourceRefCount, + boolean ownerObserved, + ReturnValueCompletion completion) { + PhysicalServerResourceAssignmentVO updated = assignments.updateFailureStatus( + assignment, isUnsupported(reasonCode) ? "UNSUPPORTED" : "ERROR", + providerType, ZBS_PROVIDER_BACKEND, + reasonCode, sourceRefCount, ownerObserved); + if (updated == null) { + reconcileZbsLatest(assignment.getServerUuid(), completion); + return; + } + completion.success(updated); + } + + private ProviderResolution resolveProvider(ZbsNodeRef ref) { + if (ref == null) { + return new ProviderResolution(null, null, "ZBS_NODE_RELATION_MISSING"); + } + List available = new ArrayList<>(); + for (ZbsCpuIsolationProvider provider : + pluginRgty.getExtensionList(ZbsCpuIsolationProvider.class)) { + if (provider.isAvailable(ref)) { + available.add(provider); + } + } + if (available.isEmpty()) { + return new ProviderResolution(null, null, "ZBS_PROVIDER_UNAVAILABLE"); + } + if (available.size() > 1) { + return new ProviderResolution(null, null, "ZBS_PROVIDER_AMBIGUOUS"); + } + ZbsCpuIsolationProvider provider = available.get(0); + if (!provider.supportsMonotonicRevisionFence()) { + return new ProviderResolution( + null, + provider.getProviderType(), + "PROVIDER_REVISION_FENCE_UNSUPPORTED"); + } + return new ProviderResolution(provider, provider.getProviderType(), null); + } + + private void refreshHostCapacity(String serverUuid) { + for (PhysicalServerCpuConsumerAdapter adapter : consumerAdapters().values()) { + try { + adapter.refreshCapacity(serverUuid); + } catch (RuntimeException error) { + logger.warn(String.format( + "failed to request capacity refresh from CPU consumer adapter[%s] " + + "for physical server[uuid:%s]: %s", + adapter.getAssignmentType(), serverUuid, error.getMessage())); + } + } + } + + private Map consumerAdapters() { + Map result = new HashMap<>(); + for (PhysicalServerCpuConsumerAdapter adapter : pluginRgty.getExtensionList( + PhysicalServerCpuConsumerAdapter.class)) { + result.put(adapter.getAssignmentType(), adapter); + } + return result; + } + + private Map> assignmentsByServer( + Collection assignmentVos) { + Map> result = new HashMap<>(); + for (PhysicalServerResourceAssignmentVO vo : assignmentVos) { + PhysicalServerCpuAssignmentType type; + try { + type = PhysicalServerCpuAssignmentType.valueOf(vo.getAssignmentType()); + } catch (IllegalArgumentException error) { + logger.warn(String.format( + "ignoring physical server CPU assignment[uuid:%s] with unsupported assignmentType[%s]", + vo.getUuid(), vo.getAssignmentType())); + continue; + } + result.computeIfAbsent(vo.getServerUuid(), ignored -> new HashMap<>()) + .put(type, vo); + } + return result; + } + + private boolean isUnsupported(String reasonCode) { + return "ZBS_PROVIDER_UNAVAILABLE".equals(reasonCode) + || "PROVIDER_VERSION_UNSUPPORTED".equals(reasonCode) + || "PROVIDER_REVISION_FENCE_UNSUPPORTED".equals(reasonCode); + } + + private static class ProviderResolution { + private final ZbsCpuIsolationProvider provider; + private final String providerType; + private final String reasonCode; + + private ProviderResolution( + ZbsCpuIsolationProvider provider, String providerType, String reasonCode) { + this.provider = provider; + this.providerType = providerType; + this.reasonCode = reasonCode; + } + } + + private static class ValidatedFact { + private final PhysicalServerCpuAssignmentStatus status; + private final Timestamp observedAt; + + private ValidatedFact( + PhysicalServerCpuAssignmentStatus status, Timestamp observedAt) { + this.status = status; + this.observedAt = observedAt; + } + } + + private static class ReconcileQueueState { + private final AtomicBoolean scheduled = new AtomicBoolean(); + private final AtomicBoolean dirty = new AtomicBoolean(); + private final Set activeSharedReleases = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final AtomicReference> assignmentSnapshot = + new AtomicReference<>(); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuSelectionType.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuSelectionType.java new file mode 100644 index 00000000000..55023d8e0a3 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerCpuSelectionType.java @@ -0,0 +1,7 @@ +package org.zstack.physicalserver; + +public enum PhysicalServerCpuSelectionType { + PERCENTAGE, + COUNT, + CPU_SET +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerGlobalConfig.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerGlobalConfig.java new file mode 100644 index 00000000000..13fb539abbc --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerGlobalConfig.java @@ -0,0 +1,17 @@ +package org.zstack.physicalserver; + +import org.zstack.core.config.GlobalConfig; +import org.zstack.core.config.GlobalConfigDef; +import org.zstack.core.config.GlobalConfigDefinition; +import org.zstack.core.config.GlobalConfigValidation; + +@GlobalConfigDefinition +public class PhysicalServerGlobalConfig { + public static final String CATEGORY = "physicalServer"; + + @GlobalConfigDef(type = Long.class, defaultValue = "60", + description = "ZBS CPU isolation Provider call timeout in seconds") + @GlobalConfigValidation(numberGreaterThan = 0) + public static GlobalConfig ZBS_PROVIDER_CALL_TIMEOUT = + new GlobalConfig(CATEGORY, "zbsCpuIsolation.providerCallTimeout"); +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerIdentityService.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerIdentityService.java new file mode 100644 index 00000000000..2f206308293 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerIdentityService.java @@ -0,0 +1,190 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.core.Platform; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import javax.persistence.Tuple; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PhysicalServerIdentityService { + private static final CLogger logger = Utils.getLogger(PhysicalServerIdentityService.class); + + @Autowired + private DatabaseFacade dbf; + + @Transactional + public Map resolve(Collection identities) { + Map requestedZones = normalize(identities); + if (requestedZones.isEmpty()) { + return Collections.emptyMap(); + } + + Map servers = serversBySerial(requestedZones.keySet()); + Set missing = new LinkedHashSet<>(requestedZones.keySet()); + missing.removeAll(servers.keySet()); + insertServers(missing); + servers = serversBySerial(requestedZones.keySet()); + assignMissingZones(servers, requestedZones); + servers = serversBySerial(requestedZones.keySet()); + + Map result = new LinkedHashMap<>(); + for (Map.Entry requested : requestedZones.entrySet()) { + IdentityRow server = servers.get(requested.getKey()); + if (server == null || (requested.getValue() != null + && !requested.getValue().equals(server.zoneUuid))) { + logger.warn(String.format( + "cannot resolve physical server identity for serialNumber[%s], " + + "reasonCode[IDENTITY_ZONE_CONFLICT]: serverZoneUuid[%s], requestedZoneUuid[%s]", + requested.getKey(), server == null ? null : server.zoneUuid, + requested.getValue())); + continue; + } + result.put(requested.getKey(), server.uuid); + } + return result; + } + + private Map normalize(Collection identities) { + if (identities == null || identities.isEmpty()) { + return Collections.emptyMap(); + } + Map requestedZones = new LinkedHashMap<>(); + Set conflicts = new LinkedHashSet<>(); + for (PhysicalServerIdentitySpec identity : identities) { + if (identity == null) { + continue; + } + String serialNumber = Platform.normalizeMachineSerialNumber(identity.getSerialNumber()); + if (serialNumber == null) { + continue; + } + String previousZone = requestedZones.get(serialNumber); + String requestedZone = identity.getZoneUuid(); + if (previousZone != null && requestedZone != null && !previousZone.equals(requestedZone)) { + conflicts.add(serialNumber); + continue; + } + if (!requestedZones.containsKey(serialNumber) || previousZone == null) { + requestedZones.put(serialNumber, requestedZone); + } + } + for (String serialNumber : conflicts) { + requestedZones.remove(serialNumber); + logger.warn(String.format( + "cannot resolve physical server identity for serialNumber[%s], " + + "reasonCode[IDENTITY_AMBIGUOUS]: multiple zones reported", + serialNumber)); + } + return requestedZones; + } + + private Map serversBySerial(Collection serialNumbers) { + if (serialNumbers.isEmpty()) { + return Collections.emptyMap(); + } + Map result = new HashMap<>(); + List servers = Q.New(PhysicalServerVO.class) + .select(PhysicalServerVO_.uuid, PhysicalServerVO_.serialNumber, + PhysicalServerVO_.zoneUuid) + .in(PhysicalServerVO_.serialNumber, serialNumbers) + .listTuple(); + for (Tuple server : servers) { + IdentityRow row = new IdentityRow( + server.get(0, String.class), server.get(2, String.class)); + result.put(server.get(1, String.class), row); + } + return result; + } + + private void insertServers(Collection serialNumbers) { + if (serialNumbers.isEmpty()) { + return; + } + StringBuilder sql = new StringBuilder( + "INSERT IGNORE INTO PhysicalServerVO " + + "(uuid, serialNumber, createDate, lastOpDate) VALUES "); + int index = 0; + for (String ignored : serialNumbers) { + if (index > 0) { + sql.append(','); + } + sql.append("(:uuid").append(index).append(", :serial").append(index) + .append(", NOW(), NOW())"); + index++; + } + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + index = 0; + for (String serialNumber : serialNumbers) { + query.setParameter("uuid" + index, Platform.getUuid()); + query.setParameter("serial" + index, serialNumber); + index++; + } + query.executeUpdate(); + } + + private void assignMissingZones( + Map servers, + Map requestedZones) { + Map zonesByServer = new LinkedHashMap<>(); + for (Map.Entry requested : requestedZones.entrySet()) { + IdentityRow server = servers.get(requested.getKey()); + if (server != null && server.zoneUuid == null + && requested.getValue() != null) { + zonesByServer.put(server.uuid, requested.getValue()); + } + } + if (zonesByServer.isEmpty()) { + return; + } + + StringBuilder sql = new StringBuilder( + "UPDATE PhysicalServerVO SET zoneUuid = CASE uuid "); + int index = 0; + for (String ignored : zonesByServer.keySet()) { + sql.append("WHEN :uuid").append(index) + .append(" THEN :zone").append(index).append(' '); + index++; + } + sql.append("END WHERE zoneUuid IS NULL AND uuid IN ("); + for (index = 0; index < zonesByServer.size(); index++) { + if (index > 0) { + sql.append(','); + } + sql.append(":uuid").append(index); + } + sql.append(')'); + + Query query = dbf.getEntityManager().createNativeQuery(sql.toString()); + index = 0; + for (Map.Entry entry : zonesByServer.entrySet()) { + query.setParameter("uuid" + index, entry.getKey()); + query.setParameter("zone" + index, entry.getValue()); + index++; + } + query.executeUpdate(); + } + + private static class IdentityRow { + private final String uuid; + private final String zoneUuid; + + private IdentityRow(String uuid, String zoneUuid) { + this.uuid = uuid; + this.zoneUuid = zoneUuid; + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventory.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventory.java new file mode 100644 index 00000000000..ac633d7c4c3 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventory.java @@ -0,0 +1,88 @@ +package org.zstack.physicalserver; + +import org.zstack.header.configuration.PythonClassInventory; +import org.zstack.header.search.Inventory; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Inventory(mappingVOClass = PhysicalServerVO.class) +@PythonClassInventory +public class PhysicalServerInventory implements Serializable { + private String uuid; + private String zoneUuid; + private String poolUuid; + private String serialNumber; + private Timestamp createDate; + private Timestamp lastOpDate; + + public static PhysicalServerInventory valueOf(PhysicalServerVO vo) { + PhysicalServerInventory inventory = new PhysicalServerInventory(); + inventory.setUuid(vo.getUuid()); + inventory.setZoneUuid(vo.getZoneUuid()); + inventory.setPoolUuid(vo.getPoolUuid()); + inventory.setSerialNumber(vo.getSerialNumber()); + inventory.setCreateDate(vo.getCreateDate()); + inventory.setLastOpDate(vo.getLastOpDate()); + return inventory; + } + + public static List valueOf(Collection vos) { + List inventories = new ArrayList<>(vos.size()); + for (PhysicalServerVO vo : vos) { + inventories.add(valueOf(vo)); + } + return inventories; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getZoneUuid() { + return zoneUuid; + } + + public void setZoneUuid(String zoneUuid) { + this.zoneUuid = zoneUuid; + } + + public String getPoolUuid() { + return poolUuid; + } + + public void setPoolUuid(String poolUuid) { + this.poolUuid = poolUuid; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventoryDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventoryDoc_zh_cn.groovy new file mode 100644 index 00000000000..9e04a711b57 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerInventoryDoc_zh_cn.groovy @@ -0,0 +1,45 @@ +package org.zstack.physicalserver + +import java.sql.Timestamp + +doc { + + title "物理服务器清单" + + field { + name "uuid" + desc "资源的UUID，唯一标示该资源" + type "String" + since "5.5.38" + } + field { + name "zoneUuid" + desc "区域UUID" + type "String" + since "5.5.38" + } + field { + name "poolUuid" + desc "所属资源池UUID；未划分资源池时为空" + type "String" + since "5.5.38" + } + field { + name "serialNumber" + desc "经标准化的服务器序列号，在云平台范围内唯一" + type "String" + since "5.5.38" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.5.38" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.5.38" + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagerImpl.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagerImpl.java new file mode 100644 index 00000000000..2f503063471 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerManagerImpl.java @@ -0,0 +1,290 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.MessageSafe; +import org.zstack.core.cloudbus.ResourceDestinationMaker; +import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.core.workflow.FlowChainBuilder; +import org.zstack.header.AbstractService; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.workflow.FlowChain; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +public class PhysicalServerManagerImpl extends AbstractService implements + PhysicalServerManager, + Component, + ManagementNodeReadyExtensionPoint { + @Autowired + private CloudBus bus; + @Autowired + private PhysicalServerIdentityService identity; + @Autowired + private PhysicalServerAssignmentRepository assignments; + @Autowired + private PhysicalServerCpuReconciler reconciler; + @Autowired + private ThreadFacade thdf; + @Autowired + private ResourceDestinationMaker destinationMaker; + private Future reconcileTask; + + @Override + @MessageSafe + public void handleMessage(Message msg) { + if (msg instanceof APIMessage) { + handleApiMessage((APIMessage) msg); + } else if (msg instanceof ReconcilePhysicalServerMsg) { + ReconcilePhysicalServerMsg rmsg = (ReconcilePhysicalServerMsg) msg; + reconciler.enqueue(rmsg.getServerUuid(), rmsg.isRefreshFacts()); + } else if (msg instanceof ReconcileAllPhysicalServersMsg) { + reconciler.enqueueAll(); + } else if (msg instanceof ReleasePhysicalServerZbsCpuIsolationMsg) { + handle((ReleasePhysicalServerZbsCpuIsolationMsg) msg); + } else if (msg instanceof ReleasePhysicalServerCpuAssignmentMsg) { + handle((ReleasePhysicalServerCpuAssignmentMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle(ReleasePhysicalServerZbsCpuIsolationMsg msg) { + ReleasePhysicalServerZbsCpuIsolationReply reply = + new ReleasePhysicalServerZbsCpuIsolationReply(); + reconciler.releaseZbsAssignment(msg.getServerUuid(), new Completion(msg) { + @Override + public void success() { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(ReleasePhysicalServerCpuAssignmentMsg msg) { + MessageReply reply = new MessageReply(); + reconciler.releaseAssignment( + msg.getServerUuid(), + PhysicalServerCpuAssignmentType.valueOf(msg.getAssignmentType()), + msg.getConsumerUuid(), + new Completion(msg) { + @Override + public void success() { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handleApiMessage(APIMessage msg) { + if (msg instanceof APIUpdatePhysicalServerCpuAssignmentMsg) { + handle((APIUpdatePhysicalServerCpuAssignmentMsg) msg); + } else if (msg instanceof APIRefreshPhysicalServerCpuIsolationMsg) { + handle((APIRefreshPhysicalServerCpuIsolationMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle(APIUpdatePhysicalServerCpuAssignmentMsg msg) { + APIUpdatePhysicalServerCpuAssignmentEvent event = + new APIUpdatePhysicalServerCpuAssignmentEvent(msg.getId()); + String inventoryKey = "updated-assignment"; + FlowChain chain = FlowChainBuilder.newSimpleFlowChain(); + chain.setName(String.format( + "update-physical-server-cpu-assignment-%s-%s", + msg.getServerUuid(), msg.getAssignmentType())); + chain.then(new NoRollbackFlow() { + String __name__ = "validate-cpu-assignment-against-current-topology"; + + @Override + public void run(FlowTrigger trigger, Map data) { + reconciler.validateUpdate(msg, new Completion(trigger) { + @Override + public void success() { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + }).then(new NoRollbackFlow() { + String __name__ = "persist-cpu-assignment-spec"; + + @Override + public void run(FlowTrigger trigger, Map data) { + PhysicalServerResourceAssignmentVO vo = assignments.updateCpu(msg); + data.put(inventoryKey, + PhysicalServerResourceAssignmentInventory.valueOf(vo)); + trigger.next(); + } + }).done(new FlowDoneHandler(msg) { + @Override + public void handle(Map data) { + event.setInventory((PhysicalServerResourceAssignmentInventory) + data.get(inventoryKey)); + reconcile(msg.getServerUuid(), true); + bus.publish(event); + } + }).error(new FlowErrorHandler(msg) { + @Override + public void handle(ErrorCode errorCode, Map data) { + event.setError(errorCode); + bus.publish(event); + } + }).start(); + } + + private void handle(APIRefreshPhysicalServerCpuIsolationMsg msg) { + APIRefreshPhysicalServerCpuIsolationEvent event = + new APIRefreshPhysicalServerCpuIsolationEvent(msg.getId()); + reconcile(msg.getServerUuid(), true); + bus.publish(event); + } + + @Override + public Map resolveIdentities( + Collection identities) { + return identity.resolve(identities); + } + + @Override + public void ensureCpuAssignments( + Collection serverUuids, String assignmentType) { + assignments.ensureDefaults( + serverUuids, PhysicalServerCpuAssignmentType.valueOf(assignmentType)); + } + + @Override + public void reconcile(String serverUuid, boolean refreshFacts) { + if (serverUuid == null) { + return; + } + ReconcilePhysicalServerMsg msg = new ReconcilePhysicalServerMsg(); + msg.setServerUuid(serverUuid); + msg.setRefreshFacts(refreshFacts); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg); + } + + @Override + public void reconcileAll() { + ReconcileAllPhysicalServersMsg msg = new ReconcileAllPhysicalServersMsg(); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg); + } + + @Override + public void releaseCpuAssignment( + String serverUuid, + String assignmentType, + String consumerUuid, + Completion completion) { + ReleasePhysicalServerCpuAssignmentMsg msg = + new ReleasePhysicalServerCpuAssignmentMsg(); + msg.setServerUuid(serverUuid); + msg.setAssignmentType(assignmentType); + msg.setConsumerUuid(consumerUuid); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + completion.success(); + } + }); + } + + @Override + public boolean start() { + return true; + } + + @Override + public void managementNodeReady() { + reconcileAllIfOwned(); + reconcileTask = thdf.submitPeriodicTask(new PeriodicTask() { + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.MILLISECONDS; + } + + @Override + public long getInterval() { + return PhysicalServerConstant.STATUS_TTL_MILLIS / 3; + } + + @Override + public String getName() { + return "physical-server-resource-assignment-reconcile"; + } + + @Override + public void run() { + reconcileAllIfOwned(); + } + }); + } + + private void reconcileAllIfOwned() { + if (destinationMaker.isManagedByUs(PhysicalServerConstant.CONTROL_OWNER_KEY)) { + reconcileAll(); + } + } + + @Override + public boolean stop() { + if (reconcileTask != null) { + reconcileTask.cancel(true); + } + return true; + } + + @Override + public String getId() { + return bus.makeLocalServiceId(PhysicalServerConstant.SERVICE_ID); + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerMessage.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerMessage.java new file mode 100644 index 00000000000..b0162c74c43 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerMessage.java @@ -0,0 +1,5 @@ +package org.zstack.physicalserver; + +public interface PhysicalServerMessage { + String getServerUuid(); +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerReasonCodes.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerReasonCodes.java new file mode 100644 index 00000000000..bbaf90d5fc0 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerReasonCodes.java @@ -0,0 +1,27 @@ +package org.zstack.physicalserver; + +import org.zstack.header.errorcode.ErrorCode; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class PhysicalServerReasonCodes { + private static final Pattern REASON_CODE = Pattern.compile( + "(?:^|[^A-Z0-9_])([A-Z][A-Z0-9_]+):"); + + private PhysicalServerReasonCodes() { + } + + static String from(ErrorCode errorCode) { + String details = errorCode == null ? null : errorCode.getDetails(); + if (details == null) { + return "CPU_ASSIGNMENT_INVALID"; + } + Matcher matcher = REASON_CODE.matcher(details); + String reasonCode = null; + while (matcher.find()) { + reasonCode = matcher.group(1); + } + return reasonCode == null ? "CPU_ASSIGNMENT_INVALID" : reasonCode; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentHandler.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentHandler.java new file mode 100644 index 00000000000..149d8263faf --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentHandler.java @@ -0,0 +1,11 @@ +package org.zstack.physicalserver; + +public interface PhysicalServerResourceAssignmentHandler { + String getResourceType(); + + boolean supportsAssignmentType(String assignmentType); + + Class getSpecClass(); + + Class getStatusClass(); +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventory.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventory.java new file mode 100644 index 00000000000..8260cc7b396 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventory.java @@ -0,0 +1,356 @@ +package org.zstack.physicalserver; + +import org.zstack.header.configuration.PythonClassInventory; +import org.zstack.header.search.Inventory; +import org.zstack.utils.gson.JSONObjectUtil; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Inventory(mappingVOClass = PhysicalServerResourceAssignmentVO.class) +@PythonClassInventory +public class PhysicalServerResourceAssignmentInventory implements Serializable { + private String uuid; + private String serverUuid; + private String resourceType; + private String assignmentType; + private String spec; + private long specGeneration; + private String status; + private Long observedGeneration; + private Timestamp observedAt; + private Timestamp createDate; + private Timestamp lastOpDate; + private String displayState; + private String displayReasonCode; + private String desiredCpuSet; + private String actualCpuSet; + private String providerType; + private String backend; + private Long dataAgeSeconds; + private Integer sourceRefCount; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private Integer requestedCount; + private Integer effectiveCount; + private Boolean clamped; + private List consumerKeys; + private List unresolvedHandles; + + public static PhysicalServerResourceAssignmentInventory valueOf(PhysicalServerResourceAssignmentVO vo) { + PhysicalServerResourceAssignmentInventory inventory = new PhysicalServerResourceAssignmentInventory(); + inventory.setUuid(vo.getUuid()); + inventory.setServerUuid(vo.getServerUuid()); + inventory.setResourceType(vo.getResourceType()); + inventory.setAssignmentType(vo.getAssignmentType()); + inventory.setSpec(vo.getSpec()); + inventory.setSpecGeneration(vo.getSpecGeneration()); + inventory.setStatus(vo.getStatus()); + inventory.setObservedGeneration(vo.getObservedGeneration()); + inventory.setObservedAt(vo.getObservedAt()); + inventory.setCreateDate(vo.getCreateDate()); + inventory.setLastOpDate(vo.getLastOpDate()); + if (!PhysicalServerConstant.CPU_RESOURCE_TYPE.equals(vo.getResourceType())) { + return inventory; + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + vo.getSpec(), PhysicalServerCpuAssignmentSpec.class); + inventory.setDesiredCpuSet(spec.getDesiredCpuSet()); + if (vo.getStatus() == null) { + inventory.setDisplayState("PENDING"); + inventory.setDisplayReasonCode("NEVER_OBSERVED"); + return inventory; + } + PhysicalServerCpuAssignmentStatus status = JSONObjectUtil.toObject( + vo.getStatus(), PhysicalServerCpuAssignmentStatus.class); + inventory.setActualCpuSet(status.getActualCpuSet()); + inventory.setProviderType(status.getProviderType()); + inventory.setBackend(status.getBackend()); + inventory.setSourceRefCount(status.getSourceRefCount()); + inventory.setCoveredServiceCount(status.getCoveredServiceCount()); + inventory.setExpectedServiceCount(status.getExpectedServiceCount()); + inventory.setRequestedCount(status.getRequestedCount()); + inventory.setEffectiveCount(status.getEffectiveCount()); + inventory.setClamped(status.getClamped()); + inventory.setConsumerKeys(status.getConsumerKeys()); + inventory.setUnresolvedHandles(status.getUnresolvedHandles()); + inventory.setDisplayReasonCode(status.getReasonCode()); + if (vo.getObservedAt() != null) { + inventory.setDataAgeSeconds(Math.max(0, + (System.currentTimeMillis() - vo.getObservedAt().getTime()) / 1000)); + } + deriveState(inventory, spec, status, vo); + return inventory; + } + + private static void deriveState( + PhysicalServerResourceAssignmentInventory inventory, + PhysicalServerCpuAssignmentSpec spec, + PhysicalServerCpuAssignmentStatus status, + PhysicalServerResourceAssignmentVO vo) { + boolean stale = vo.getObservedAt() == null + || System.currentTimeMillis() - vo.getObservedAt().getTime() + > PhysicalServerConstant.STATUS_TTL_MILLIS + || isOwnerUnreachable(status.getReasonCode()); + if (stale) { + inventory.setDisplayState("STALE"); + if (inventory.getDisplayReasonCode() == null) { + inventory.setDisplayReasonCode("OWNER_FACT_STALE"); + } + return; + } + if ("UNSUPPORTED".equals(status.getOwnerState())) { + inventory.setDisplayState("UNSUPPORTED"); + return; + } + if ("ERROR".equals(status.getOwnerState())) { + inventory.setDisplayState("CONFLICT"); + return; + } + if (vo.getObservedGeneration() == null + || vo.getObservedGeneration() != vo.getSpecGeneration() + || "PENDING".equals(status.getOwnerState())) { + inventory.setDisplayState("PENDING"); + return; + } + boolean matches = spec.isEnabled() + ? "READY".equals(status.getOwnerState()) + && equal(spec.getDesiredCpuSet(), status.getActualCpuSet()) + : "DISABLED".equals(status.getOwnerState()) + && (status.getActualCpuSet() == null || status.getActualCpuSet().isEmpty()); + inventory.setDisplayState(matches ? "READY" : "CONFLICT"); + if (!matches && inventory.getDisplayReasonCode() == null) { + inventory.setDisplayReasonCode("ACTUAL_MISMATCH"); + } + } + + private static boolean isOwnerUnreachable(String reasonCode) { + return "OWNER_UNREACHABLE".equals(reasonCode) + || "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE".equals(reasonCode) + || "SHARED_CPUSET_EXECUTOR_UNREACHABLE".equals(reasonCode) + || "CPU_CONSUMER_UNAVAILABLE".equals(reasonCode); + } + + private static boolean equal(Object left, Object right) { + return left == null ? right == null : left.equals(right); + } + + public static List valueOf(Collection vos) { + List inventories = new ArrayList<>(vos.size()); + for (PhysicalServerResourceAssignmentVO vo : vos) { + inventories.add(valueOf(vo)); + } + return inventories; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getAssignmentType() { + return assignmentType; + } + + public void setAssignmentType(String assignmentType) { + this.assignmentType = assignmentType; + } + + public String getSpec() { + return spec; + } + + public void setSpec(String spec) { + this.spec = spec; + } + + public long getSpecGeneration() { + return specGeneration; + } + + public void setSpecGeneration(long specGeneration) { + this.specGeneration = specGeneration; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Long getObservedGeneration() { + return observedGeneration; + } + + public void setObservedGeneration(Long observedGeneration) { + this.observedGeneration = observedGeneration; + } + + public Timestamp getObservedAt() { + return observedAt; + } + + public void setObservedAt(Timestamp observedAt) { + this.observedAt = observedAt; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + public String getDisplayState() { + return displayState; + } + + public void setDisplayState(String displayState) { + this.displayState = displayState; + } + + public String getDisplayReasonCode() { + return displayReasonCode; + } + + public void setDisplayReasonCode(String displayReasonCode) { + this.displayReasonCode = displayReasonCode; + } + + public String getDesiredCpuSet() { + return desiredCpuSet; + } + + public void setDesiredCpuSet(String desiredCpuSet) { + this.desiredCpuSet = desiredCpuSet; + } + + public String getActualCpuSet() { + return actualCpuSet; + } + + public void setActualCpuSet(String actualCpuSet) { + this.actualCpuSet = actualCpuSet; + } + + public String getProviderType() { + return providerType; + } + + public void setProviderType(String providerType) { + this.providerType = providerType; + } + + public String getBackend() { + return backend; + } + + public void setBackend(String backend) { + this.backend = backend; + } + + public Long getDataAgeSeconds() { + return dataAgeSeconds; + } + + public void setDataAgeSeconds(Long dataAgeSeconds) { + this.dataAgeSeconds = dataAgeSeconds; + } + + public Integer getSourceRefCount() { + return sourceRefCount; + } + + public void setSourceRefCount(Integer sourceRefCount) { + this.sourceRefCount = sourceRefCount; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public Integer getRequestedCount() { + return requestedCount; + } + + public void setRequestedCount(Integer requestedCount) { + this.requestedCount = requestedCount; + } + + public Integer getEffectiveCount() { + return effectiveCount; + } + + public void setEffectiveCount(Integer effectiveCount) { + this.effectiveCount = effectiveCount; + } + + public Boolean getClamped() { + return clamped; + } + + public void setClamped(Boolean clamped) { + this.clamped = clamped; + } + + public List getConsumerKeys() { + return consumerKeys; + } + + public void setConsumerKeys(List consumerKeys) { + this.consumerKeys = consumerKeys; + } + + public List getUnresolvedHandles() { + return unresolvedHandles; + } + + public void setUnresolvedHandles(List unresolvedHandles) { + this.unresolvedHandles = unresolvedHandles; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventoryDoc_zh_cn.groovy b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventoryDoc_zh_cn.groovy new file mode 100644 index 00000000000..0cc63205eec --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentInventoryDoc_zh_cn.groovy @@ -0,0 +1,168 @@ +package org.zstack.physicalserver + +import java.lang.Long +import java.sql.Timestamp +import java.lang.Integer +import java.lang.Boolean + +doc { + + title "物理服务器资源分配清单" + + field { + name "uuid" + desc "资源的UUID，唯一标示该资源" + type "String" + since "5.5.38" + } + field { + name "serverUuid" + desc "物理服务器UUID" + type "String" + since "5.5.38" + } + field { + name "resourceType" + desc "资源类型，当前为CPU" + type "String" + since "5.5.38" + } + field { + name "assignmentType" + desc "分配类型" + type "String" + since "5.5.38" + } + field { + name "spec" + desc "期望配置的版本化JSON" + type "String" + since "5.5.38" + } + field { + name "specGeneration" + desc "期望配置版本；每次更新单调递增" + type "long" + since "5.5.38" + } + field { + name "status" + desc "最近一次探测到的实际状态JSON；从未探测时为空" + type "String" + since "5.5.38" + } + field { + name "observedGeneration" + desc "最近一次实际状态对应的期望配置版本" + type "Long" + since "5.5.38" + } + field { + name "observedAt" + desc "最近一次实际状态采集时间" + type "Timestamp" + since "5.5.38" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.5.38" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.5.38" + } + field { + name "displayState" + desc "根据期望和实际状态计算的展示态：PENDING、READY、CONFLICT、UNSUPPORTED或STALE" + type "String" + since "5.5.38" + } + field { + name "displayReasonCode" + desc "展示态原因码" + type "String" + since "5.5.38" + } + field { + name "desiredCpuSet" + desc "云平台计划使用的CPU集合" + type "String" + since "5.5.38" + } + field { + name "actualCpuSet" + desc "最近一次由执行方报告的CPU集合" + type "String" + since "5.5.38" + } + field { + name "providerType" + desc "执行或探测该分配的Provider类型" + type "String" + since "5.5.38" + } + field { + name "backend" + desc "实际使用的隔离或亲和后端" + type "String" + since "5.5.38" + } + field { + name "dataAgeSeconds" + desc "实际状态距当前时间的秒数" + type "Long" + since "5.5.38" + } + field { + name "sourceRefCount" + desc "推导该Consumer的业务关系数量" + type "Integer" + since "5.5.38" + } + field { + name "coveredServiceCount" + desc "已成功应用或探测的服务实例数量" + type "Integer" + since "5.5.38" + } + field { + name "expectedServiceCount" + desc "应当应用或探测的服务实例数量" + type "Integer" + since "5.5.38" + } + field { + name "requestedCount" + desc "按配置申请的CPU数量" + type "Integer" + since "5.5.38" + } + field { + name "effectiveCount" + desc "结合拓扑与边界后实际规划的CPU数量" + type "Integer" + since "5.5.38" + } + field { + name "clamped" + desc "实际规划数量是否因可用CPU不足而被裁剪" + type "Boolean" + since "5.5.38" + } + field { + name "consumerKeys" + desc "被该分配覆盖的稳定Consumer标识" + type "List" + since "5.5.38" + } + field { + name "unresolvedHandles" + desc "无法解析或应用的服务句柄及原因" + type "List" + since "5.5.38" + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO.java new file mode 100644 index 00000000000..9dd7992f604 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO.java @@ -0,0 +1,152 @@ +package org.zstack.physicalserver; + +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.Index; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.PreUpdate; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import java.sql.Timestamp; + +@Entity +@Table(uniqueConstraints = @UniqueConstraint( + name = "ukPhysicalServerResourceAssignment", + columnNames = {"serverUuid", "resourceType", "assignmentType"} +)) +public class PhysicalServerResourceAssignmentVO { + @Id + @Column + @Index + private String uuid; + + @Column + @Index + @ForeignKey(parentEntityClass = PhysicalServerVO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String serverUuid; + + @Column + private String resourceType; + + @Column + private String assignmentType; + + @Lob + @Column + private String spec; + + @Column + private long specGeneration; + + @Lob + @Column + private String status; + + @Column + private Long observedGeneration; + + @Column + private Timestamp observedAt; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @PreUpdate + private void preUpdate() { + lastOpDate = null; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getAssignmentType() { + return assignmentType; + } + + public void setAssignmentType(String assignmentType) { + this.assignmentType = assignmentType; + } + + public String getSpec() { + return spec; + } + + public void setSpec(String spec) { + this.spec = spec; + } + + public long getSpecGeneration() { + return specGeneration; + } + + public void setSpecGeneration(long specGeneration) { + this.specGeneration = specGeneration; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Long getObservedGeneration() { + return observedGeneration; + } + + public void setObservedGeneration(Long observedGeneration) { + this.observedGeneration = observedGeneration; + } + + public Timestamp getObservedAt() { + return observedAt; + } + + public void setObservedAt(Timestamp observedAt) { + this.observedAt = observedAt; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO_.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO_.java new file mode 100644 index 00000000000..48d7782b447 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerResourceAssignmentVO_.java @@ -0,0 +1,20 @@ +package org.zstack.physicalserver; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(PhysicalServerResourceAssignmentVO.class) +public class PhysicalServerResourceAssignmentVO_ { + public static volatile SingularAttribute uuid; + public static volatile SingularAttribute serverUuid; + public static volatile SingularAttribute resourceType; + public static volatile SingularAttribute assignmentType; + public static volatile SingularAttribute spec; + public static volatile SingularAttribute specGeneration; + public static volatile SingularAttribute status; + public static volatile SingularAttribute observedGeneration; + public static volatile SingularAttribute observedAt; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerSharedCpuReconciler.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerSharedCpuReconciler.java new file mode 100644 index 00000000000..9d7763d7053 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerSharedCpuReconciler.java @@ -0,0 +1,659 @@ +package org.zstack.physicalserver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerAdapter; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerState; +import org.zstack.header.physicalserver.PhysicalServerCpuSet; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.SharedCpuSetCommand; +import org.zstack.header.physicalserver.SharedCpuSetResponse; +import org.zstack.utils.Utils; +import org.zstack.utils.gson.JSONObjectUtil; +import org.zstack.utils.logging.CLogger; + +import java.sql.Timestamp; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.zstack.core.Platform.operr; + +public class PhysicalServerSharedCpuReconciler { + private static final CLogger logger = Utils.getLogger( + PhysicalServerSharedCpuReconciler.class); + + @Autowired + private PluginRegistry pluginRgty; + @Autowired + private PhysicalServerAssignmentRepository assignments; + @Autowired + private PhysicalServerCpuPlanner planner; + + public void reconcile(String serverUuid, Completion completion) { + reconcile(serverUuid, null, null, false, completion); + } + + public void reconcile( + String serverUuid, + Map assignmentSnapshot, + PhysicalServerResourceAssignmentVO zbsAssignment, + boolean skipCompute, + Completion completion) { + PhysicalServerResourceAssignmentVO managementAssignment = assignment( + serverUuid, + PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED, + assignmentSnapshot); + if (skipCompute) { + reconcileType( + serverUuid, + PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED, + managementAssignment, + zbsAssignment, + managementAssignment, + completion); + return; + } + reconcileType( + serverUuid, + PhysicalServerCpuAssignmentType.COMPUTE_SHARED, + assignment( + serverUuid, + PhysicalServerCpuAssignmentType.COMPUTE_SHARED, + assignmentSnapshot), + zbsAssignment, + managementAssignment, + new Completion(completion) { + @Override + public void success() { + reconcileType( + serverUuid, + PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED, + managementAssignment, + zbsAssignment, + managementAssignment, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to reconcile compute shared CPUSet for " + + "physical server[uuid:%s]: %s", + serverUuid, errorCode)); + reconcileType( + serverUuid, + PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED, + managementAssignment, + zbsAssignment, + managementAssignment, + completion); + } + }); + } + + public void release( + String serverUuid, + PhysicalServerCpuAssignmentType type, + String consumerUuid, + Completion completion) { + PhysicalServerResourceAssignmentVO assignment = assignment(serverUuid, type); + if (skip(assignment)) { + completion.success(); + return; + } + PhysicalServerCpuConsumerAdapter adapter = adapters().get(type.name()); + if (adapter == null) { + releaseFailure(assignment, null, "CPU_CONSUMER_ADAPTER_MISSING", completion); + return; + } + + SharedCpuSetCommand command = command( + type, + assignment, + assignment(serverUuid, PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED)); + command.setEnabled(false); + adapter.apply( + serverUuid, + consumerUuid, + command, + new ReturnValueCompletion(completion) { + @Override + public void success(SharedCpuSetResponse response) { + String actual = response.getActualCpuSet() == null + ? "" : response.getActualCpuSet().trim(); + boolean released = actual.isEmpty() + && response.getExpectedServiceCount() != null + && response.getExpectedServiceCount().equals( + response.getCoveredServiceCount()); + if (!released) { + releaseFailure( + assignment, + response.getBackend(), + response.getReasonCode() == null + ? "SHARED_CPUSET_RELEASE_FAILED" + : response.getReasonCode(), + completion); + return; + } + recordReleased(assignment, type, response, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + releaseFailure( + assignment, + null, + errorCode.getDetails(), + completion); + } + }); + } + + private void reconcileType( + String serverUuid, + PhysicalServerCpuAssignmentType type, + Completion completion) { + PhysicalServerResourceAssignmentVO managementAssignment = assignment( + serverUuid, PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED); + reconcileType( + serverUuid, + type, + assignment(serverUuid, type), + assignment(serverUuid, PhysicalServerCpuAssignmentType.ZBS_EXCLUSIVE), + managementAssignment, + completion); + } + + private void reconcileType( + String serverUuid, + PhysicalServerCpuAssignmentType type, + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerResourceAssignmentVO zbsAssignment, + PhysicalServerResourceAssignmentVO managementAssignment, + Completion completion) { + if (skip(assignment)) { + completion.success(); + return; + } + PhysicalServerCpuConsumerAdapter adapter = adapters().get(type.name()); + if (adapter == null) { + persistFailure( + assignment, + "UNSUPPORTED", + null, + "CPU_CONSUMER_ADAPTER_MISSING", + true, + completion); + return; + } + + PhysicalServerCpuConsumerState state = adapter.getState(serverUuid); + if (state == PhysicalServerCpuConsumerState.MISSING) { + if (releaseFailed(assignment)) { + completion.success(); + return; + } + persistFailure( + assignment, + "UNSUPPORTED", + null, + missingReason(type), + true, + completion); + return; + } + if (state == PhysicalServerCpuConsumerState.UNAVAILABLE) { + persistFailure( + assignment, + "ERROR", + null, + unavailableReason(type), + false, + completion); + return; + } + + adapter.collectTopology( + serverUuid, + new ReturnValueCompletion(completion) { + @Override + public void success(PhysicalServerCpuTopology topology) { + planAndApply( + assignment, + type, + adapter, + topology, + zbsAssignment, + managementAssignment, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + assignment, + "ERROR", + null, + "CPU_TOPOLOGY_EXECUTOR_UNREACHABLE", + false, + completion); + } + }); + } + + private void planAndApply( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuAssignmentType type, + PhysicalServerCpuConsumerAdapter adapter, + PhysicalServerCpuTopology topology, + PhysicalServerResourceAssignmentVO zbsAssignment, + PhysicalServerResourceAssignmentVO managementAssignment, + Completion completion) { + plan(assignment, topology, zbsAssignment, + new ReturnValueCompletion(completion) { + @Override + public void success(PlannedAssignment planned) { + applyPlanned( + assignment, + type, + adapter, + topology, + managementAssignment, + planned, + completion); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + assignment, + "ERROR", + null, + PhysicalServerReasonCodes.from(errorCode), + true, + completion); + } + }); + } + + private void applyPlanned( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuAssignmentType type, + PhysicalServerCpuConsumerAdapter adapter, + PhysicalServerCpuTopology topology, + PhysicalServerResourceAssignmentVO managementAssignment, + PlannedAssignment planned, + Completion completion) { + if (planned.retry) { + reconcileType(assignment.getServerUuid(), type, completion); + return; + } + + SharedCpuSetCommand command = command( + type, planned.assignment, managementAssignment); + adapter.apply( + assignment.getServerUuid(), + null, + command, + new ReturnValueCompletion(completion) { + @Override + public void success(SharedCpuSetResponse response) { + record( + planned, + topology, + response, + completion, + () -> reconcileType( + assignment.getServerUuid(), type, completion)); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + planned.assignment, + "ERROR", + null, + "SHARED_CPUSET_EXECUTOR_UNREACHABLE", + false, + completion); + } + }); + } + + private void plan( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuTopology topology, + PhysicalServerResourceAssignmentVO zbsAssignment, + ReturnValueCompletion completion) { + PlannedAssignment planned; + try { + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + PhysicalServerCpuPlan plan = planner.plan( + PhysicalServerCpuAssignmentType.valueOf(assignment.getAssignmentType()), + spec, + topology, + exclusiveCpus(zbsAssignment, topology)); + PhysicalServerResourceAssignmentVO updated = assignments.updatePlan(assignment, plan); + if (updated == null + || updated.getSpecGeneration() != assignment.getSpecGeneration()) { + planned = new PlannedAssignment(null, plan, true); + } else { + planned = new PlannedAssignment(updated, plan, false); + } + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, "%s", error.getMessage())); + return; + } + completion.success(planned); + } + + private Set exclusiveCpus( + PhysicalServerResourceAssignmentVO zbs, + PhysicalServerCpuTopology topology) { + if (zbs == null) { + return Collections.emptySet(); + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + zbs.getSpec(), PhysicalServerCpuAssignmentSpec.class); + String actual = null; + if (zbs.getStatus() != null) { + actual = JSONObjectUtil.toObject( + zbs.getStatus(), PhysicalServerCpuAssignmentStatus.class) + .getActualCpuSet(); + } + String conservative = PhysicalServerCpuSet.union( + spec.getDesiredCpuSet(), actual); + return conservative.isEmpty() + ? Collections.emptySet() + : PhysicalServerCpuSet.parse( + conservative, topology.getOnlineCpus()); + } + + private SharedCpuSetCommand command( + PhysicalServerCpuAssignmentType type, + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerResourceAssignmentVO managementAssignment) { + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + SharedCpuSetCommand command = new SharedCpuSetCommand(); + command.setRoleKey(type.name().toLowerCase()); + command.setCpuSet(spec.getDesiredCpuSet()); + command.setEnabled(spec.isEnabled()); + command.setIncludeAuxiliaryServices( + type == PhysicalServerCpuAssignmentType.MANAGEMENT_SHARED + || !managementOwnsNodeAux(managementAssignment)); + return command; + } + + private void record( + PlannedAssignment planned, + PhysicalServerCpuTopology topology, + SharedCpuSetResponse response, + Completion completion, + Runnable retry) { + if (response == null) { + persistFailure( + planned.assignment, + "ERROR", + null, + "SHARED_CPUSET_FACT_INVALID", + true, + completion); + return; + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + planned.assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + normalizeActualCpuSet( + response, + topology, + new ReturnValueCompletion(completion) { + @Override + public void success(String actual) { + recordNormalized( + planned, spec, response, actual, completion, retry); + } + + @Override + public void fail(ErrorCode errorCode) { + persistFailure( + planned.assignment, + "ERROR", + response.getBackend(), + "SHARED_CPUSET_FACT_INVALID", + true, + completion); + } + }); + } + + private void normalizeActualCpuSet( + SharedCpuSetResponse response, + PhysicalServerCpuTopology topology, + ReturnValueCompletion completion) { + String actual; + try { + actual = response.getActualCpuSet() == null + || response.getActualCpuSet().trim().isEmpty() + ? "" + : PhysicalServerCpuSet.format(PhysicalServerCpuSet.parse( + response.getActualCpuSet(), topology.getOnlineCpus())); + } catch (RuntimeException error) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, "%s", error.getMessage())); + return; + } + completion.success(actual); + } + + private void recordNormalized( + PlannedAssignment planned, + PhysicalServerCpuAssignmentSpec spec, + SharedCpuSetResponse response, + String actual, + Completion completion, + Runnable retry) { + boolean coverageReady = response.getExpectedServiceCount() != null + && response.getExpectedServiceCount() > 0 + && response.getExpectedServiceCount().equals( + response.getCoveredServiceCount()); + boolean matches = spec.isEnabled() + ? coverageReady && spec.getDesiredCpuSet().equals(actual) + : coverageReady && actual.isEmpty(); + + String reasonCode = matches + ? null + : response.getReasonCode() == null + ? "ACTUAL_MISMATCH" : response.getReasonCode(); + PhysicalServerCpuAssignmentStatus status = new PhysicalServerCpuAssignmentStatus(); + status.setOwnerState(matches + ? spec.isEnabled() ? "READY" : "DISABLED" + : isUnsupported(reasonCode) ? "UNSUPPORTED" : "ERROR"); + status.setConfiguredCpuSet(actual); + status.setActualCpuSet(actual); + status.setAppliedPlanHash(matches ? spec.getPlanHash() : null); + status.setBackend(response.getBackend()); + status.setReasonCode(reasonCode); + status.setCoveredServiceCount(response.getCoveredServiceCount()); + status.setExpectedServiceCount(response.getExpectedServiceCount()); + status.setRequestedCount(planned.plan.getRequestedCount()); + status.setEffectiveCount(planned.plan.getEffectiveCount()); + status.setClamped(planned.plan.isClamped()); + status.setConsumerKeys(response.getConsumerKeys()); + status.setUnresolvedHandles(response.getUnresolvedHandles()); + PhysicalServerResourceAssignmentVO updated = assignments.updateStatus( + planned.assignment, + status, + new Timestamp(System.currentTimeMillis())); + if (updated == null) { + retry.run(); + return; + } + completion.success(); + } + + private void recordReleased( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuAssignmentType type, + SharedCpuSetResponse response, + Completion completion) { + PhysicalServerCpuAssignmentStatus status = assignment.getStatus() == null + ? new PhysicalServerCpuAssignmentStatus() + : JSONObjectUtil.toObject( + assignment.getStatus(), PhysicalServerCpuAssignmentStatus.class); + status.setOwnerState("UNSUPPORTED"); + status.setConfiguredCpuSet(""); + status.setActualCpuSet(""); + status.setAppliedPlanHash(null); + status.setBackend(response.getBackend()); + status.setReasonCode(missingReason(type)); + status.setCoveredServiceCount(response.getCoveredServiceCount()); + status.setExpectedServiceCount(response.getExpectedServiceCount()); + status.setConsumerKeys(response.getConsumerKeys()); + status.setUnresolvedHandles(response.getUnresolvedHandles()); + PhysicalServerResourceAssignmentVO updated = assignments.updateStatus( + assignment, + status, + new Timestamp(System.currentTimeMillis())); + if (updated == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "shared CPU assignment changed while releasing physical server[uuid:%s]", + assignment.getServerUuid())); + return; + } + completion.success(); + } + + private void releaseFailure( + PhysicalServerResourceAssignmentVO assignment, + String backend, + String details, + Completion completion) { + assignments.updateFailureStatus( + assignment, + "ERROR", + null, + backend, + "SHARED_CPUSET_RELEASE_FAILED", + null, + true); + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "failed to release shared CPUSet on physical server[uuid:%s]: %s", + assignment.getServerUuid(), details)); + } + + private void persistFailure( + PhysicalServerResourceAssignmentVO assignment, + String ownerState, + String backend, + String reasonCode, + boolean ownerObserved, + Completion completion) { + PhysicalServerResourceAssignmentVO updated = assignments.updateFailureStatus( + assignment, + ownerState, + null, + backend, + reasonCode, + null, + ownerObserved); + if (updated == null) { + completion.fail(operr( + PhysicalServerConstant.ERROR_CODE, + "shared CPU assignment changed while persisting status")); + return; + } + completion.success(); + } + + private Map adapters() { + Map result = new HashMap<>(); + List extensions = pluginRgty.getExtensionList( + PhysicalServerCpuConsumerAdapter.class); + for (PhysicalServerCpuConsumerAdapter extension : extensions) { + result.put(extension.getAssignmentType(), extension); + } + return result; + } + + private PhysicalServerResourceAssignmentVO assignment( + String serverUuid, + PhysicalServerCpuAssignmentType type) { + return assignments.find(serverUuid, type); + } + + private PhysicalServerResourceAssignmentVO assignment( + String serverUuid, + PhysicalServerCpuAssignmentType type, + Map assignmentSnapshot) { + return assignmentSnapshot == null + ? assignment(serverUuid, type) : assignmentSnapshot.get(type); + } + + private boolean managementOwnsNodeAux( + PhysicalServerResourceAssignmentVO assignment) { + if (assignment == null) { + return false; + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + return !"AWAITING_DEFAULT_FREEZE".equals(spec.getSource()); + } + + private boolean skip(PhysicalServerResourceAssignmentVO assignment) { + if (assignment == null) { + return true; + } + PhysicalServerCpuAssignmentSpec spec = JSONObjectUtil.toObject( + assignment.getSpec(), PhysicalServerCpuAssignmentSpec.class); + return !spec.isEnabled() + && "AWAITING_DEFAULT_FREEZE".equals(spec.getSource()); + } + + private boolean releaseFailed(PhysicalServerResourceAssignmentVO assignment) { + if (assignment.getStatus() == null) { + return false; + } + PhysicalServerCpuAssignmentStatus status = JSONObjectUtil.toObject( + assignment.getStatus(), PhysicalServerCpuAssignmentStatus.class); + return "SHARED_CPUSET_RELEASE_FAILED".equals(status.getReasonCode()); + } + + private String missingReason(PhysicalServerCpuAssignmentType type) { + return "CPU_CONSUMER_RELATION_MISSING"; + } + + private String unavailableReason(PhysicalServerCpuAssignmentType type) { + return "CPU_CONSUMER_UNAVAILABLE"; + } + + private boolean isUnsupported(String reasonCode) { + return "SHARED_CPUSET_UNAVAILABLE".equals(reasonCode); + } + + private static class PlannedAssignment { + private final PhysicalServerResourceAssignmentVO assignment; + private final PhysicalServerCpuPlan plan; + private final boolean retry; + + private PlannedAssignment( + PhysicalServerResourceAssignmentVO assignment, + PhysicalServerCpuPlan plan, + boolean retry) { + this.assignment = assignment; + this.plan = plan; + this.retry = retry; + } + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO.java new file mode 100644 index 00000000000..d13aefb8718 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO.java @@ -0,0 +1,88 @@ +package org.zstack.physicalserver; + +import org.zstack.header.vo.Index; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.PreUpdate; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; +import java.sql.Timestamp; + +@Entity +@Table(uniqueConstraints = @UniqueConstraint(name = "ukPhysicalServerSerialNumber", columnNames = "serialNumber")) +public class PhysicalServerVO { + @Id + @Column + @Index + private String uuid; + + @Column + private String zoneUuid; + + @Column + private String poolUuid; + + @Column + private String serialNumber; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @PreUpdate + private void preUpdate() { + lastOpDate = null; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getZoneUuid() { + return zoneUuid; + } + + public void setZoneUuid(String zoneUuid) { + this.zoneUuid = zoneUuid; + } + + public String getPoolUuid() { + return poolUuid; + } + + public void setPoolUuid(String poolUuid) { + this.poolUuid = poolUuid; + } + + public String getSerialNumber() { + return serialNumber; + } + + public void setSerialNumber(String serialNumber) { + this.serialNumber = serialNumber; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO_.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO_.java new file mode 100644 index 00000000000..95271a7bd84 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/PhysicalServerVO_.java @@ -0,0 +1,15 @@ +package org.zstack.physicalserver; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(PhysicalServerVO.class) +public class PhysicalServerVO_ { + public static volatile SingularAttribute uuid; + public static volatile SingularAttribute zoneUuid; + public static volatile SingularAttribute poolUuid; + public static volatile SingularAttribute serialNumber; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/RBACInfo.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/RBACInfo.java new file mode 100644 index 00000000000..2f060e28e08 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/RBACInfo.java @@ -0,0 +1,30 @@ +package org.zstack.physicalserver; + +import org.zstack.header.identity.rbac.RBACDescription; + +public class RBACInfo implements RBACDescription { + @Override + public void permissions() { + permissionBuilder() + .name("physicalServer") + .adminOnlyAPIs("org.zstack.physicalserver.**") + .build(); + } + + @Override + public void contributeToRoles() { + } + + @Override + public void roles() { + roleBuilder() + .name("physicalServer") + .uuid("f8d781036ed84b69a9737ec6d78e65e1") + .permissionsByName("physicalServer") + .build(); + } + + @Override + public void globalReadableResources() { + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcileAllPhysicalServersMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcileAllPhysicalServersMsg.java new file mode 100644 index 00000000000..51ef0b3f5a6 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcileAllPhysicalServersMsg.java @@ -0,0 +1,6 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.Message; + +public class ReconcileAllPhysicalServersMsg extends Message { +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcilePhysicalServerMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcilePhysicalServerMsg.java new file mode 100644 index 00000000000..65960a53cd8 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReconcilePhysicalServerMsg.java @@ -0,0 +1,25 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.Message; + +public class ReconcilePhysicalServerMsg extends Message implements PhysicalServerMessage { + private String serverUuid; + private boolean refreshFacts; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public boolean isRefreshFacts() { + return refreshFacts; + } + + public void setRefreshFacts(boolean refreshFacts) { + this.refreshFacts = refreshFacts; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerCpuAssignmentMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerCpuAssignmentMsg.java new file mode 100644 index 00000000000..b54b7c0674f --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerCpuAssignmentMsg.java @@ -0,0 +1,35 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.NeedReplyMessage; + +public class ReleasePhysicalServerCpuAssignmentMsg extends NeedReplyMessage + implements PhysicalServerMessage { + private String serverUuid; + private String assignmentType; + private String consumerUuid; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public String getAssignmentType() { + return assignmentType; + } + + public void setAssignmentType(String assignmentType) { + this.assignmentType = assignmentType; + } + + public String getConsumerUuid() { + return consumerUuid; + } + + public void setConsumerUuid(String consumerUuid) { + this.consumerUuid = consumerUuid; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerZbsCpuIsolationMsg.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerZbsCpuIsolationMsg.java new file mode 100644 index 00000000000..3ef575b609e --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerZbsCpuIsolationMsg.java @@ -0,0 +1,16 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.NeedReplyMessage; + +public class ReleasePhysicalServerZbsCpuIsolationMsg extends NeedReplyMessage implements PhysicalServerMessage { + private String serverUuid; + + @Override + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerZbsCpuIsolationReply.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerZbsCpuIsolationReply.java new file mode 100644 index 00000000000..0c6a0ad3494 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ReleasePhysicalServerZbsCpuIsolationReply.java @@ -0,0 +1,6 @@ +package org.zstack.physicalserver; + +import org.zstack.header.message.MessageReply; + +public class ReleasePhysicalServerZbsCpuIsolationReply extends MessageReply { +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationFact.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationFact.java new file mode 100644 index 00000000000..4d9746fb059 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationFact.java @@ -0,0 +1,105 @@ +package org.zstack.physicalserver; + +import java.sql.Timestamp; + +public class ZbsCpuIsolationFact { + private String capabilityVersion; + private String isolationState; + private String configuredCpuSet; + private String effectiveCpuSet; + private Integer coveredServiceCount; + private Integer expectedServiceCount; + private boolean serviceReady; + private String ownerAssignmentUuid; + private String ownerGeneration; + private Timestamp observedAt; + private String reasonCode; + + public String getCapabilityVersion() { + return capabilityVersion; + } + + public void setCapabilityVersion(String capabilityVersion) { + this.capabilityVersion = capabilityVersion; + } + + public String getIsolationState() { + return isolationState; + } + + public void setIsolationState(String isolationState) { + this.isolationState = isolationState; + } + + public String getConfiguredCpuSet() { + return configuredCpuSet; + } + + public void setConfiguredCpuSet(String configuredCpuSet) { + this.configuredCpuSet = configuredCpuSet; + } + + public String getEffectiveCpuSet() { + return effectiveCpuSet; + } + + public void setEffectiveCpuSet(String effectiveCpuSet) { + this.effectiveCpuSet = effectiveCpuSet; + } + + public Integer getCoveredServiceCount() { + return coveredServiceCount; + } + + public void setCoveredServiceCount(Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + + public Integer getExpectedServiceCount() { + return expectedServiceCount; + } + + public void setExpectedServiceCount(Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + + public boolean isServiceReady() { + return serviceReady; + } + + public void setServiceReady(boolean serviceReady) { + this.serviceReady = serviceReady; + } + + public String getOwnerAssignmentUuid() { + return ownerAssignmentUuid; + } + + public void setOwnerAssignmentUuid(String ownerAssignmentUuid) { + this.ownerAssignmentUuid = ownerAssignmentUuid; + } + + public String getOwnerGeneration() { + return ownerGeneration; + } + + public void setOwnerGeneration(String ownerGeneration) { + this.ownerGeneration = ownerGeneration; + } + + public Timestamp getObservedAt() { + return observedAt; + } + + public void setObservedAt(Timestamp observedAt) { + this.observedAt = observedAt; + } + + public String getReasonCode() { + return reasonCode; + } + + public void setReasonCode(String reasonCode) { + this.reasonCode = reasonCode; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationProvider.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationProvider.java new file mode 100644 index 00000000000..7585c00f2b2 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationProvider.java @@ -0,0 +1,22 @@ +package org.zstack.physicalserver; + +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; + +public interface ZbsCpuIsolationProvider { + String getProviderType(); + + boolean isAvailable(ZbsNodeRef nodeRef); + + boolean supportsMonotonicRevisionFence(); + + void query(ZbsNodeRef nodeRef, ReturnValueCompletion completion); + + /** + * The provider must durably fence updates by assignmentUuid and revision. Revisions are + * monotonic within one assignmentUuid, and the same revision is idempotent only for the + * same payload. A new assignmentUuid can start with SET only after the previous assignment + * has reached DISABLED. Query reports both accepted values after the request is effective. + */ + void update(ZbsNodeRef nodeRef, ZbsCpuIsolationUpdate update, Completion completion); +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationUpdate.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationUpdate.java new file mode 100644 index 00000000000..1cced2f27b3 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsCpuIsolationUpdate.java @@ -0,0 +1,49 @@ +package org.zstack.physicalserver; + +public class ZbsCpuIsolationUpdate { + private String assignmentUuid; + private String operation; + private String desiredCpuSet; + private String planHash; + private long revision; + + public String getAssignmentUuid() { + return assignmentUuid; + } + + public void setAssignmentUuid(String assignmentUuid) { + this.assignmentUuid = assignmentUuid; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getDesiredCpuSet() { + return desiredCpuSet; + } + + public void setDesiredCpuSet(String desiredCpuSet) { + this.desiredCpuSet = desiredCpuSet; + } + + public String getPlanHash() { + return planHash; + } + + public void setPlanHash(String planHash) { + this.planHash = planHash; + } + + public long getRevision() { + return revision; + } + + public void setRevision(long revision) { + this.revision = revision; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsNodeRef.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsNodeRef.java new file mode 100644 index 00000000000..3d527022bce --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsNodeRef.java @@ -0,0 +1,64 @@ +package org.zstack.physicalserver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +public class ZbsNodeRef { + private String serverUuid; + private final Set primaryStorageUuids = new LinkedHashSet<>(); + private final Set nodeAddresses = new LinkedHashSet<>(); + private int sourceRefCount; + private String reasonCode; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public List getPrimaryStorageUuids() { + return new ArrayList<>(primaryStorageUuids); + } + + public void addPrimaryStorageUuid(String primaryStorageUuid) { + primaryStorageUuids.add(primaryStorageUuid); + } + + public List getNodeAddresses() { + return new ArrayList<>(nodeAddresses); + } + + public void addNodeAddress(String nodeAddress) { + nodeAddresses.add(nodeAddress); + } + + public int getSourceRefCount() { + return sourceRefCount; + } + + public void incrementSourceRefCount() { + sourceRefCount++; + } + + public String getReasonCode() { + return reasonCode; + } + + public void setReasonCode(String reasonCode) { + this.reasonCode = reasonCode; + } + + public boolean includesAnyPrimaryStorage(Collection uuids) { + for (String uuid : uuids) { + if (primaryStorageUuids.contains(uuid)) { + return true; + } + } + return false; + } +} diff --git a/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsNodeRefContributor.java b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsNodeRefContributor.java new file mode 100644 index 00000000000..43d14081e19 --- /dev/null +++ b/plugin/physicalServer/src/main/java/org/zstack/physicalserver/ZbsNodeRefContributor.java @@ -0,0 +1,8 @@ +package org.zstack.physicalserver; + +import java.util.Collection; +import java.util.Map; + +public interface ZbsNodeRefContributor { + Map bulkList(Collection serverUuids); +} diff --git a/plugin/pom.xml b/plugin/pom.xml index 22f39138fc7..32e060799d3 100755 --- a/plugin/pom.xml +++ b/plugin/pom.xml @@ -33,6 +33,7 @@ directory sshKeyPair hostNetworkInterface + physicalServer vhost expon externalStorage diff --git a/plugin/zbs/pom.xml b/plugin/zbs/pom.xml index c1a8f8e2a85..d4ca5b9b733 100644 --- a/plugin/zbs/pom.xml +++ b/plugin/zbs/pom.xml @@ -11,6 +11,11 @@ zbs + + org.zstack + physicalServer + ${project.version} + org.zstack cbd @@ -90,4 +95,4 @@ - \ No newline at end of file + diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationCascadeExtension.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationCascadeExtension.java new file mode 100644 index 00000000000..d8343163d2a --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsCpuIsolationCascadeExtension.java @@ -0,0 +1,229 @@ +package org.zstack.storage.zbs; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.EventCallback; +import org.zstack.core.cloudbus.EventFacade; +import org.zstack.core.db.Q; +import org.zstack.header.Component; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_; +import org.zstack.header.storage.primary.PrimaryStorageCanonicalEvent; +import org.zstack.header.storage.primary.PrimaryStorageInventory; +import org.zstack.header.storage.primary.PrimaryStorageVO; +import org.zstack.physicalserver.PhysicalServerConstant; +import org.zstack.physicalserver.ReconcileAllPhysicalServersMsg; +import org.zstack.physicalserver.ReleasePhysicalServerZbsCpuIsolationMsg; +import org.zstack.physicalserver.ZbsNodeRef; +import org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.zstack.core.Platform.operr; + +public class ZbsCpuIsolationCascadeExtension extends AbstractAsyncCascadeExtension implements Component { + private static final String NAME = "ZbsCpuIsolationAssignment"; + + @Autowired + private CloudBus bus; + @Autowired + private EventFacade evtf; + @Autowired + private ZbsNodeRefContributorImpl nodeRefs; + + private EventCallback addonInfoChanged; + private EventCallback primaryStorageDeleted; + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (!PrimaryStorageVO.class.getSimpleName().equals(action.getParentIssuer()) + || !CascadeConstant.DELETION_CODES.contains(action.getActionCode())) { + completion.success(); + return; + } + + Set deleting = deletingZbsPrimaryStorages(action); + if (deleting.isEmpty()) { + completion.success(); + return; + } + + List lastRelationServers = lastRelationServers(deleting); + if (action.isActionCode(CascadeConstant.DELETION_CHECK_CODE) + || lastRelationServers.isEmpty()) { + completion.success(); + return; + } + + release(lastRelationServers, completion); + } + + public void beforePersistAddonInfo( + String primaryStorageUuid, + AddonInfo addonInfo, + Completion completion) { + Set newRelationServers = + nodeRefs.serverUuidsForAddonInfo(primaryStorageUuid, addonInfo); + String oldAddonInfo = Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.addonInfo) + .eq(ExternalPrimaryStorageVO_.uuid, primaryStorageUuid) + .findValue(); + if (oldAddonInfo == null || oldAddonInfo.isEmpty()) { + completion.success(); + return; + } + List removedLastRelations = lastRelationServers( + Collections.singleton(primaryStorageUuid)); + removedLastRelations.removeAll(newRelationServers); + release(removedLastRelations, completion); + } + + private void release(List serverUuids, Completion completion) { + if (serverUuids.isEmpty()) { + completion.success(); + return; + } + new While<>(serverUuids).each((serverUuid, each) -> { + ReleasePhysicalServerZbsCpuIsolationMsg msg = + new ReleasePhysicalServerZbsCpuIsolationMsg(); + msg.setServerUuid(serverUuid); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg, new CloudBusCallBack(each) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + each.addError(reply.getError()); + each.allDone(); + return; + } + each.done(); + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(errorCodeList.getCauses().get(0)); + return; + } + completion.success(); + } + }); + } + + private Set deletingZbsPrimaryStorages(CascadeAction action) { + List inventories = action.getParentIssuerContext(); + if (inventories == null || inventories.isEmpty()) { + return Collections.emptySet(); + } + List candidates = new ArrayList<>(); + for (PrimaryStorageInventory inventory : inventories) { + candidates.add(inventory.getUuid()); + } + return new HashSet<>(Q.New(ExternalPrimaryStorageVO.class) + .select(ExternalPrimaryStorageVO_.uuid) + .in(ExternalPrimaryStorageVO_.uuid, candidates) + .eq(ExternalPrimaryStorageVO_.identity, ZbsConstants.IDENTITY) + .listValues()); + } + + private List lastRelationServers(Set deletingPrimaryStorageUuids) { + Map refs = nodeRefs.bulkList(Collections.emptySet()); + List result = new ArrayList<>(); + for (ZbsNodeRef ref : refs.values()) { + if (!ref.includesAnyPrimaryStorage(deletingPrimaryStorageUuids)) { + continue; + } + if (ref.getReasonCode() != null) { + throw new OperationFailureException(operr( + PhysicalServerConstant.ERROR_CODE, + "cannot determine the last ZBS relationship for physical server[uuid:%s], reason[%s]", + ref.getServerUuid(), ref.getReasonCode())); + } + boolean hasRemainingRelationship = ref.getPrimaryStorageUuids().stream() + .anyMatch(uuid -> !deletingPrimaryStorageUuids.contains(uuid)); + if (!hasRemainingRelationship) { + result.add(ref.getServerUuid()); + } + } + return result; + } + + private void sendReconcileAll() { + ReconcileAllPhysicalServersMsg msg = new ReconcileAllPhysicalServersMsg(); + bus.makeTargetServiceIdByResourceUuid( + msg, + PhysicalServerConstant.SERVICE_ID, + PhysicalServerConstant.CONTROL_OWNER_KEY); + bus.send(msg); + } + + @Override + public boolean start() { + addonInfoChanged = new EventCallback() { + @Override + protected void run( + Map tokens, + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData data) { + sendReconcileAll(); + } + }; + primaryStorageDeleted = new EventCallback() { + @Override + protected void run( + Map tokens, + PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData data) { + sendReconcileAll(); + } + }; + evtf.on(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, addonInfoChanged); + evtf.on(PrimaryStorageCanonicalEvent.PRIMARY_STORAGE_DELETED_PATH, primaryStorageDeleted); + return true; + } + + @Override + public boolean stop() { + if (addonInfoChanged != null) { + evtf.off(addonInfoChanged); + } + if (primaryStorageDeleted != null) { + evtf.off(primaryStorageDeleted); + } + return true; + } + + @Override + public List getEdgeNames() { + return Collections.singletonList(PrimaryStorageVO.class.getSimpleName()); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + return null; + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributorImpl.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributorImpl.java new file mode 100644 index 00000000000..c7209082e0d --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsNodeRefContributorImpl.java @@ -0,0 +1,172 @@ +package org.zstack.storage.zbs; + +import org.zstack.compute.host.HostSystemTags; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostVO_; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO; +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_; +import org.zstack.header.tag.SystemTagVO; +import org.zstack.header.tag.SystemTagVO_; +import org.zstack.kvm.KVMHostVO; +import org.zstack.physicalserver.ZbsNodeRef; +import org.zstack.physicalserver.ZbsNodeRefContributor; +import org.zstack.physicalserver.PhysicalServerConstant; +import org.zstack.utils.TagUtils; +import org.zstack.utils.gson.JSONObjectUtil; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.zstack.core.Platform.operr; + +public class ZbsNodeRefContributorImpl implements ZbsNodeRefContributor { + @Override + public Map bulkList(Collection serverUuids) { + List hosts = listHosts(serverUuids); + if (hosts.isEmpty()) { + return Collections.emptyMap(); + } + + Map> serversByIp = mapServersByIp(hosts); + List primaryStorages = Q.New(ExternalPrimaryStorageVO.class) + .eq(ExternalPrimaryStorageVO_.identity, ZbsConstants.IDENTITY) + .list(); + Map result = new LinkedHashMap<>(); + for (ExternalPrimaryStorageVO primaryStorage : primaryStorages) { + AddonInfo addonInfo = parseAddonInfo(primaryStorage); + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds == null) { + throw invalidAddonInfo(primaryStorage.getUuid(), + "contains an empty mdsInfo"); + } + String address = resolveAddress(mds, serversByIp); + if (address == null) { + continue; + } + Set matchedServers = serversByIp.get(address); + for (String serverUuid : matchedServers) { + ZbsNodeRef ref = result.computeIfAbsent(serverUuid, ignored -> { + ZbsNodeRef created = new ZbsNodeRef(); + created.setServerUuid(serverUuid); + return created; + }); + ref.addPrimaryStorageUuid(primaryStorage.getUuid()); + ref.addNodeAddress(address); + ref.incrementSourceRefCount(); + if (matchedServers.size() > 1) { + ref.setReasonCode("ZBS_NODE_ADDRESS_AMBIGUOUS"); + } + } + } + } + return result; + } + + Set serverUuidsForAddonInfo( + String primaryStorageUuid, AddonInfo addonInfo) { + validateAddonInfo(primaryStorageUuid, addonInfo); + Map> serversByIp = mapServersByIp( + listHosts(Collections.emptySet())); + Set result = new HashSet<>(); + for (MdsInfo mds : addonInfo.getMdsInfos()) { + if (mds == null) { + throw invalidAddonInfo(primaryStorageUuid, "contains an empty mdsInfo"); + } + String address = resolveAddress(mds, serversByIp); + if (address != null) { + result.addAll(serversByIp.get(address)); + } + } + return result; + } + + private List listHosts(Collection serverUuids) { + Q hostQuery = Q.New(KVMHostVO.class) + .notNull(HostVO_.serverUuid); + if (serverUuids != null && !serverUuids.isEmpty()) { + hostQuery.in(HostVO_.serverUuid, serverUuids); + } + return hostQuery.list(); + } + + private AddonInfo parseAddonInfo(ExternalPrimaryStorageVO primaryStorage) { + if (primaryStorage.getAddonInfo() == null || primaryStorage.getAddonInfo().isEmpty()) { + throw invalidAddonInfo(primaryStorage.getUuid(), "is empty"); + } + + AddonInfo addonInfo = JSONObjectUtil.toObject( + primaryStorage.getAddonInfo(), AddonInfo.class); + validateAddonInfo(primaryStorage.getUuid(), addonInfo); + return addonInfo; + } + + private void validateAddonInfo(String primaryStorageUuid, AddonInfo addonInfo) { + if (addonInfo == null || addonInfo.getMdsInfos() == null + || addonInfo.getMdsInfos().isEmpty()) { + throw invalidAddonInfo(primaryStorageUuid, "does not contain mdsInfos"); + } + } + + private OperationFailureException invalidAddonInfo(String primaryStorageUuid, String detail) { + return new OperationFailureException(operr( + PhysicalServerConstant.ERROR_CODE, + "ZBS_NODE_RELATION_SOURCE_INVALID: cannot derive ZBS node relations because " + + "primary storage[uuid:%s] addonInfo %s", + primaryStorageUuid, + detail)); + } + + private Map> mapServersByIp(List hosts) { + Map> result = new HashMap<>(); + if (hosts.isEmpty()) { + return result; + } + Map serverByHost = new HashMap<>(); + for (KVMHostVO host : hosts) { + serverByHost.put(host.getUuid(), host.getServerUuid()); + add(result, host.getManagementIp(), host.getServerUuid()); + } + + List tags = Q.New(SystemTagVO.class) + .in(SystemTagVO_.resourceUuid, serverByHost.keySet()) + .eq(SystemTagVO_.resourceType, org.zstack.header.host.HostVO.class.getSimpleName()) + .like(SystemTagVO_.tag, + TagUtils.tagPatternToSqlPattern(HostSystemTags.EXTRA_IPS.getTagFormat())) + .list(); + for (SystemTagVO tag : tags) { + String extraIps = HostSystemTags.EXTRA_IPS.getTokenByTag( + tag.getTag(), HostSystemTags.EXTRA_IPS_TOKEN); + if (extraIps == null) { + continue; + } + for (String ip : extraIps.split(",")) { + add(result, ip.trim(), serverByHost.get(tag.getResourceUuid())); + } + } + return result; + } + + private String resolveAddress(MdsInfo mds, Map> serversByIp) { + if (mds.getExternalAddr() != null && serversByIp.containsKey(mds.getExternalAddr())) { + return mds.getExternalAddr(); + } + if (mds.getAddr() != null && serversByIp.containsKey(mds.getAddr())) { + return mds.getAddr(); + } + return null; + } + + private void add(Map> values, String key, String value) { + if (key == null || key.isEmpty() || value == null) { + return; + } + values.computeIfAbsent(key, ignored -> new HashSet<>()).add(value); + } +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 74386981e87..6b4bd8ced38 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -92,6 +92,8 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar @Autowired @Deprecated private CloudBus bus; + @Autowired + private ZbsCpuIsolationCascadeExtension cpuIsolationCascade; private ExternalPrimaryStorageVO self; private AddonInfo addonInfo; @@ -1650,6 +1652,21 @@ public void syncConfig(String config) { this.config = StringUtils.isEmpty(config) ? new Config() : JSONObjectUtil.toObject(config, Config.class); } + @Override + public void beforePersistAddonInfo( + org.zstack.header.storage.addon.primary.AddonInfo addonInfo, + Completion completion) { + if (!(addonInfo instanceof AddonInfo)) { + completion.fail(operr( + ORG_ZSTACK_STORAGE_ZBS_10012, + "cannot update ZBS primary storage[uuid:%s]: invalid addonInfo type[%s]", + self.getUuid(), addonInfo == null ? null : addonInfo.getClass().getName())); + return; + } + cpuIsolationCascade.beforePersistAddonInfo( + self.getUuid(), (AddonInfo) addonInfo, completion); + } + private void syncMdsStatuses(AddonInfo newAddonInfo) { if (addonInfo == null || newAddonInfo == null) { return; diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeSharedCpuSetMsg.java b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeSharedCpuSetMsg.java new file mode 100644 index 00000000000..1170173f3bb --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeSharedCpuSetMsg.java @@ -0,0 +1,25 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.physicalserver.SharedCpuSetCommand; + +public class ApplyManagementNodeSharedCpuSetMsg extends NeedReplyMessage { + private String serverUuid; + private SharedCpuSetCommand command; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } + + public SharedCpuSetCommand getCommand() { + return command; + } + + public void setCommand(SharedCpuSetCommand command) { + this.command = command; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeSharedCpuSetReply.java b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeSharedCpuSetReply.java new file mode 100644 index 00000000000..a387d28db08 --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/ApplyManagementNodeSharedCpuSetReply.java @@ -0,0 +1,16 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.SharedCpuSetResponse; + +public class ApplyManagementNodeSharedCpuSetReply extends MessageReply { + private SharedCpuSetResponse response; + + public SharedCpuSetResponse getResponse() { + return response; + } + + public void setResponse(SharedCpuSetResponse response) { + this.response = response; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyMsg.java b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyMsg.java new file mode 100644 index 00000000000..6a18ec43880 --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyMsg.java @@ -0,0 +1,15 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.NeedReplyMessage; + +public class CollectManagementNodeCpuTopologyMsg extends NeedReplyMessage { + private String serverUuid; + + public String getServerUuid() { + return serverUuid; + } + + public void setServerUuid(String serverUuid) { + this.serverUuid = serverUuid; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyReply.java b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyReply.java new file mode 100644 index 00000000000..78738ea96fa --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/CollectManagementNodeCpuTopologyReply.java @@ -0,0 +1,16 @@ +package org.zstack.portal.managementnode; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; + +public class CollectManagementNodeCpuTopologyReply extends MessageReply { + private PhysicalServerCpuTopology topology; + + public PhysicalServerCpuTopology getTopology() { + return topology; + } + + public void setTopology(PhysicalServerCpuTopology topology) { + this.topology = topology; + } +} diff --git a/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodePhysicalServerAdapter.java b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodePhysicalServerAdapter.java new file mode 100644 index 00000000000..5acceaaa106 --- /dev/null +++ b/portal/src/main/java/org/zstack/portal/managementnode/ManagementNodePhysicalServerAdapter.java @@ -0,0 +1,378 @@ +package org.zstack.portal.managementnode; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.MessageSafe; +import org.zstack.core.cpu.LocalCpuTopologyCollector; +import org.zstack.core.cpu.LocalSharedCpuSetExecutor; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.AbstractService; +import org.zstack.header.Component; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.managementnode.ManagementNodeChangeListener; +import org.zstack.header.managementnode.ManagementNodeInventory; +import org.zstack.header.managementnode.ManagementNodeReadyExtensionPoint; +import org.zstack.header.managementnode.ManagementNodeState; +import org.zstack.header.managementnode.ManagementNodeVO; +import org.zstack.header.managementnode.ManagementNodeVO_; +import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerAdapter; +import org.zstack.header.physicalserver.PhysicalServerCpuConsumerState; +import org.zstack.header.physicalserver.PhysicalServerCpuTopology; +import org.zstack.header.physicalserver.PhysicalServerIdentitySpec; +import org.zstack.header.physicalserver.PhysicalServerManager; +import org.zstack.header.physicalserver.SharedCpuSetCommand; +import org.zstack.header.physicalserver.SharedCpuSetResponse; +import org.zstack.header.physicalserver.SharedCpuSetServiceHandle; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Query; +import javax.persistence.Tuple; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.zstack.core.Platform.operr; + +public class ManagementNodePhysicalServerAdapter extends AbstractService implements + PhysicalServerCpuConsumerAdapter, + ManagementNodeChangeListener, + ManagementNodeReadyExtensionPoint, + Component { + public static final String ASSIGNMENT_TYPE = "MANAGEMENT_SHARED"; + private static final String SERVICE_ID = "managementNodePhysicalServerCpu"; + private static final String ERROR_CODE = "ORG_ZSTACK_PORTAL_10000"; + private static final CLogger logger = Utils.getLogger( + ManagementNodePhysicalServerAdapter.class); + private static final List MANAGEMENT_CORE_UNITS = Arrays.asList( + "mariadb.service", + "prometheus.service", + "pushgateway.service"); + private static final List MANAGEMENT_AUX_UNITS = Arrays.asList( + "zstack-ui-nginx.service", + "zops-ui.service", + "zops.service", + "vector.service", + "filebeat.service", + "node_exporter.service", + "collectd.service", + "collectd_exporter.service", + "process_exporter.service", + "zstack_service_exporter.service"); + + @Autowired(required = false) + private PhysicalServerManager physicalServerManager; + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + @Autowired + private LocalCpuTopologyCollector localTopology; + @Autowired + private LocalSharedCpuSetExecutor localCpuSetExecutor; + + @Override + @MessageSafe + public void handleMessage(Message msg) { + if (msg instanceof CollectManagementNodeCpuTopologyMsg) { + handle((CollectManagementNodeCpuTopologyMsg) msg); + } else if (msg instanceof ApplyManagementNodeSharedCpuSetMsg) { + handle((ApplyManagementNodeSharedCpuSetMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle(CollectManagementNodeCpuTopologyMsg msg) { + CollectManagementNodeCpuTopologyReply reply = + new CollectManagementNodeCpuTopologyReply(); + if (!owns(msg.getServerUuid())) { + reply.setError(notOwner(msg.getServerUuid())); + bus.reply(msg, reply); + return; + } + collectLocalTopology(new ReturnValueCompletion(null) { + @Override + public void success(PhysicalServerCpuTopology topology) { + reply.setTopology(topology); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(ApplyManagementNodeSharedCpuSetMsg msg) { + ApplyManagementNodeSharedCpuSetReply reply = + new ApplyManagementNodeSharedCpuSetReply(); + if (!owns(msg.getServerUuid())) { + reply.setError(notOwner(msg.getServerUuid())); + bus.reply(msg, reply); + return; + } + applyLocalCpuSet(msg.getCommand(), new ReturnValueCompletion(null) { + @Override + public void success(SharedCpuSetResponse response) { + reply.setResponse(response); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void collectLocalTopology( + ReturnValueCompletion completion) { + completion.success(localTopology.collect()); + } + + private void applyLocalCpuSet( + SharedCpuSetCommand command, + ReturnValueCompletion completion) { + command.setHandles(managementHandles(Platform.getManagementServerId())); + completion.success(localCpuSetExecutor.apply(command)); + } + + @Override + public String getAssignmentType() { + return ASSIGNMENT_TYPE; + } + + @Override + public Set getAssociatedServerUuids() { + return new LinkedHashSet<>(Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .notNull(ManagementNodeVO_.serverUuid) + .listValues()); + } + + @Override + public PhysicalServerCpuConsumerState getState(String serverUuid) { + Tuple node = Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.uuid, ManagementNodeVO_.state) + .eq(ManagementNodeVO_.serverUuid, serverUuid) + .findTuple(); + if (node == null) { + return PhysicalServerCpuConsumerState.MISSING; + } + return node.get(1, ManagementNodeState.class) == ManagementNodeState.RUNNING + ? PhysicalServerCpuConsumerState.AVAILABLE + : PhysicalServerCpuConsumerState.UNAVAILABLE; + } + + @Override + public void collectTopology( + String serverUuid, + ReturnValueCompletion completion) { + String nodeUuid = nodeUuid(serverUuid, null); + if (nodeUuid == null) { + completion.fail(notOwner(serverUuid)); + return; + } + CollectManagementNodeCpuTopologyMsg msg = + new CollectManagementNodeCpuTopologyMsg(); + msg.setServerUuid(serverUuid); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeServiceIdByManagementNodeId(msg, SERVICE_ID, nodeUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + CollectManagementNodeCpuTopologyReply topologyReply = reply.castReply(); + completion.success(topologyReply.getTopology()); + } + }); + } + + @Override + public void apply( + String serverUuid, + String consumerUuid, + SharedCpuSetCommand command, + ReturnValueCompletion completion) { + String nodeUuid = nodeUuid(serverUuid, consumerUuid); + if (nodeUuid == null) { + completion.fail(notOwner(serverUuid)); + return; + } + ApplyManagementNodeSharedCpuSetMsg msg = + new ApplyManagementNodeSharedCpuSetMsg(); + msg.setServerUuid(serverUuid); + msg.setCommand(command); + msg.setTimeout(TimeUnit.MINUTES.toMillis(5)); + bus.makeServiceIdByManagementNodeId(msg, SERVICE_ID, nodeUuid); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + ApplyManagementNodeSharedCpuSetReply applyReply = reply.castReply(); + completion.success(applyReply.getResponse()); + } + }); + } + + @Override + public void nodeJoin(ManagementNodeInventory inv) { + } + + @Override + public void nodeLeft(ManagementNodeInventory inv) { + if (physicalServerManager != null && inv.getServerUuid() != null) { + physicalServerManager.reconcile(inv.getServerUuid(), false); + } + } + + @Override + public void iAmDead(ManagementNodeInventory inv) { + } + + @Override + public void iJoin(ManagementNodeInventory inv) { + associateLocalNode(inv.getUuid()); + } + + @Override + public void managementNodeReady() { + associateLocalNode(Platform.getManagementServerId()); + } + + @Transactional + public void associateLocalNode(String nodeUuid) { + if (physicalServerManager == null + || !Platform.getManagementServerId().equals(nodeUuid)) { + return; + } + String current = Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, nodeUuid) + .findValue(); + if (current == null) { + String serialNumber = Platform.getManagementServerSerialNumber(); + if (serialNumber == null) { + logger.warn(String.format( + "cannot associate management node[uuid:%s] with a physical server, " + + "reasonCode[IDENTITY_UNAVAILABLE]", nodeUuid)); + return; + } + current = physicalServerManager.resolveIdentities( + Collections.singletonList( + new PhysicalServerIdentitySpec(serialNumber, null))) + .get(serialNumber); + if (current == null || linkedNode(current, nodeUuid) != null) { + return; + } + Query update = dbf.getEntityManager().createNativeQuery( + "UPDATE IGNORE ManagementNodeVO SET serverUuid = :serverUuid " + + "WHERE uuid = :nodeUuid AND serverUuid IS NULL"); + update.setParameter("serverUuid", current); + update.setParameter("nodeUuid", nodeUuid); + update.executeUpdate(); + current = Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, nodeUuid) + .findValue(); + } + if (current != null) { + physicalServerManager.ensureCpuAssignment(current, ASSIGNMENT_TYPE); + physicalServerManager.reconcile(current, true); + } + } + + private String linkedNode(String serverUuid, String excludedNodeUuid) { + return Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.uuid) + .eq(ManagementNodeVO_.serverUuid, serverUuid) + .notEq(ManagementNodeVO_.uuid, excludedNodeUuid) + .findValue(); + } + + private String nodeUuid(String serverUuid, String consumerUuid) { + if (consumerUuid != null) { + return Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.uuid) + .eq(ManagementNodeVO_.uuid, consumerUuid) + .eq(ManagementNodeVO_.serverUuid, serverUuid) + .findValue(); + } + return Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.uuid) + .eq(ManagementNodeVO_.serverUuid, serverUuid) + .findValue(); + } + + private boolean owns(String serverUuid) { + return serverUuid != null && serverUuid.equals(Q.New(ManagementNodeVO.class) + .select(ManagementNodeVO_.serverUuid) + .eq(ManagementNodeVO_.uuid, Platform.getManagementServerId()) + .findValue()); + } + + private ErrorCode notOwner(String serverUuid) { + return operr(ERROR_CODE, + "management node[uuid:%s] is not associated with physical server[uuid:%s]", + Platform.getManagementServerId(), serverUuid); + } + + private List managementHandles(String nodeUuid) { + String coreKey = String.format("mn-core:%s", nodeUuid); + String auxiliaryKey = String.format("mn-aux:%s", nodeUuid); + List handles = new ArrayList<>(); + handles.add(new SharedCpuSetServiceHandle( + SharedCpuSetServiceHandle.OWNER_PID_FILE, + CoreGlobalProperty.PID_FILE_PATH, coreKey, false, "java")); + for (String unit : MANAGEMENT_CORE_UNITS) { + handles.add(new SharedCpuSetServiceHandle( + SharedCpuSetServiceHandle.SYSTEMD_UNIT, + unit, coreKey, true, null)); + } + for (String unit : MANAGEMENT_AUX_UNITS) { + if (!MANAGEMENT_CORE_UNITS.contains(unit)) { + handles.add(new SharedCpuSetServiceHandle( + SharedCpuSetServiceHandle.SYSTEMD_UNIT, + unit, auxiliaryKey, true, null)); + } + } + return handles; + } + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Override + public String getId() { + return bus.makeLocalServiceId(SERVICE_ID); + } +} diff --git a/sdk/src/main/java/SourceClassMap.java b/sdk/src/main/java/SourceClassMap.java index 29c9197c37d..36b208b396a 100644 --- a/sdk/src/main/java/SourceClassMap.java +++ b/sdk/src/main/java/SourceClassMap.java @@ -693,6 +693,8 @@ public class SourceClassMap { put("org.zstack.pciDevice.virtual.vfio_mdev.MdevDeviceState", "org.zstack.sdk.MdevDeviceState"); put("org.zstack.pciDevice.virtual.vfio_mdev.MdevDeviceStatus", "org.zstack.sdk.MdevDeviceStatus"); put("org.zstack.pciDevice.virtual.vfio_mdev.MdevDeviceType", "org.zstack.sdk.MdevDeviceType"); + put("org.zstack.physicalserver.PhysicalServerInventory", "org.zstack.sdk.PhysicalServerInventory"); + put("org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory", "org.zstack.sdk.PhysicalServerResourceAssignmentInventory"); put("org.zstack.policyRoute.PolicyRouteRuleInventory", "org.zstack.sdk.PolicyRouteRuleInventory"); put("org.zstack.policyRoute.PolicyRouteRuleProtocol", "org.zstack.sdk.PolicyRouteRuleProtocol"); put("org.zstack.policyRoute.PolicyRouteRuleSetInventory", "org.zstack.sdk.PolicyRouteRuleSetInventory"); @@ -1409,6 +1411,8 @@ public class SourceClassMap { put("org.zstack.sdk.PciDeviceVirtStatus", "org.zstack.pciDevice.virtual.PciDeviceVirtStatus"); put("org.zstack.sdk.PendingTaskInfo", "org.zstack.header.core.progress.PendingTaskInfo"); put("org.zstack.sdk.PhysicalDriveSmartSelfTestHistoryInventory", "org.zstack.storage.device.localRaid.PhysicalDriveSmartSelfTestHistoryInventory"); + put("org.zstack.sdk.PhysicalServerInventory", "org.zstack.physicalserver.PhysicalServerInventory"); + put("org.zstack.sdk.PhysicalServerResourceAssignmentInventory", "org.zstack.physicalserver.PhysicalServerResourceAssignmentInventory"); put("org.zstack.sdk.PhysicalSwitchInventory", "org.zstack.network.hostNetworkInterface.PhysicalSwitchInventory"); put("org.zstack.sdk.PhysicalSwitchPortInventory", "org.zstack.network.hostNetworkInterface.PhysicalSwitchPortInventory"); put("org.zstack.sdk.PluginDriverInventory", "org.zstack.header.core.external.plugin.PluginDriverInventory"); diff --git a/sdk/src/main/java/org/zstack/sdk/HostInventory.java b/sdk/src/main/java/org/zstack/sdk/HostInventory.java index 1fce8a8b39d..3d584fb8815 100644 --- a/sdk/src/main/java/org/zstack/sdk/HostInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/HostInventory.java @@ -244,6 +244,14 @@ public java.lang.String getArchitecture() { return this.architecture; } + public java.lang.String serverUuid; + public void setServerUuid(java.lang.String serverUuid) { + this.serverUuid = serverUuid; + } + public java.lang.String getServerUuid() { + return this.serverUuid; + } + public java.sql.Timestamp createDate; public void setCreateDate(java.sql.Timestamp createDate) { this.createDate = createDate; diff --git a/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java b/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java index ab788e532f0..12857365c00 100644 --- a/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/ManagementNodeInventory.java @@ -36,4 +36,12 @@ public java.sql.Timestamp getHeartBeat() { return this.heartBeat; } + public java.lang.String serverUuid; + public void setServerUuid(java.lang.String serverUuid) { + this.serverUuid = serverUuid; + } + public java.lang.String getServerUuid() { + return this.serverUuid; + } + } diff --git a/sdk/src/main/java/org/zstack/sdk/PhysicalServerInventory.java b/sdk/src/main/java/org/zstack/sdk/PhysicalServerInventory.java new file mode 100644 index 00000000000..0ca92227250 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/PhysicalServerInventory.java @@ -0,0 +1,55 @@ +package org.zstack.sdk; + + + +public class PhysicalServerInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String zoneUuid; + public void setZoneUuid(java.lang.String zoneUuid) { + this.zoneUuid = zoneUuid; + } + public java.lang.String getZoneUuid() { + return this.zoneUuid; + } + + public java.lang.String poolUuid; + public void setPoolUuid(java.lang.String poolUuid) { + this.poolUuid = poolUuid; + } + public java.lang.String getPoolUuid() { + return this.poolUuid; + } + + public java.lang.String serialNumber; + public void setSerialNumber(java.lang.String serialNumber) { + this.serialNumber = serialNumber; + } + public java.lang.String getSerialNumber() { + return this.serialNumber; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/PhysicalServerResourceAssignmentInventory.java b/sdk/src/main/java/org/zstack/sdk/PhysicalServerResourceAssignmentInventory.java new file mode 100644 index 00000000000..2f13a5df2cb --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/PhysicalServerResourceAssignmentInventory.java @@ -0,0 +1,215 @@ +package org.zstack.sdk; + + + +public class PhysicalServerResourceAssignmentInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String serverUuid; + public void setServerUuid(java.lang.String serverUuid) { + this.serverUuid = serverUuid; + } + public java.lang.String getServerUuid() { + return this.serverUuid; + } + + public java.lang.String resourceType; + public void setResourceType(java.lang.String resourceType) { + this.resourceType = resourceType; + } + public java.lang.String getResourceType() { + return this.resourceType; + } + + public java.lang.String assignmentType; + public void setAssignmentType(java.lang.String assignmentType) { + this.assignmentType = assignmentType; + } + public java.lang.String getAssignmentType() { + return this.assignmentType; + } + + public java.lang.String spec; + public void setSpec(java.lang.String spec) { + this.spec = spec; + } + public java.lang.String getSpec() { + return this.spec; + } + + public long specGeneration; + public void setSpecGeneration(long specGeneration) { + this.specGeneration = specGeneration; + } + public long getSpecGeneration() { + return this.specGeneration; + } + + public java.lang.String status; + public void setStatus(java.lang.String status) { + this.status = status; + } + public java.lang.String getStatus() { + return this.status; + } + + public java.lang.Long observedGeneration; + public void setObservedGeneration(java.lang.Long observedGeneration) { + this.observedGeneration = observedGeneration; + } + public java.lang.Long getObservedGeneration() { + return this.observedGeneration; + } + + public java.sql.Timestamp observedAt; + public void setObservedAt(java.sql.Timestamp observedAt) { + this.observedAt = observedAt; + } + public java.sql.Timestamp getObservedAt() { + return this.observedAt; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + + public java.lang.String displayState; + public void setDisplayState(java.lang.String displayState) { + this.displayState = displayState; + } + public java.lang.String getDisplayState() { + return this.displayState; + } + + public java.lang.String displayReasonCode; + public void setDisplayReasonCode(java.lang.String displayReasonCode) { + this.displayReasonCode = displayReasonCode; + } + public java.lang.String getDisplayReasonCode() { + return this.displayReasonCode; + } + + public java.lang.String desiredCpuSet; + public void setDesiredCpuSet(java.lang.String desiredCpuSet) { + this.desiredCpuSet = desiredCpuSet; + } + public java.lang.String getDesiredCpuSet() { + return this.desiredCpuSet; + } + + public java.lang.String actualCpuSet; + public void setActualCpuSet(java.lang.String actualCpuSet) { + this.actualCpuSet = actualCpuSet; + } + public java.lang.String getActualCpuSet() { + return this.actualCpuSet; + } + + public java.lang.String providerType; + public void setProviderType(java.lang.String providerType) { + this.providerType = providerType; + } + public java.lang.String getProviderType() { + return this.providerType; + } + + public java.lang.String backend; + public void setBackend(java.lang.String backend) { + this.backend = backend; + } + public java.lang.String getBackend() { + return this.backend; + } + + public java.lang.Long dataAgeSeconds; + public void setDataAgeSeconds(java.lang.Long dataAgeSeconds) { + this.dataAgeSeconds = dataAgeSeconds; + } + public java.lang.Long getDataAgeSeconds() { + return this.dataAgeSeconds; + } + + public java.lang.Integer sourceRefCount; + public void setSourceRefCount(java.lang.Integer sourceRefCount) { + this.sourceRefCount = sourceRefCount; + } + public java.lang.Integer getSourceRefCount() { + return this.sourceRefCount; + } + + public java.lang.Integer coveredServiceCount; + public void setCoveredServiceCount(java.lang.Integer coveredServiceCount) { + this.coveredServiceCount = coveredServiceCount; + } + public java.lang.Integer getCoveredServiceCount() { + return this.coveredServiceCount; + } + + public java.lang.Integer expectedServiceCount; + public void setExpectedServiceCount(java.lang.Integer expectedServiceCount) { + this.expectedServiceCount = expectedServiceCount; + } + public java.lang.Integer getExpectedServiceCount() { + return this.expectedServiceCount; + } + + public java.lang.Integer requestedCount; + public void setRequestedCount(java.lang.Integer requestedCount) { + this.requestedCount = requestedCount; + } + public java.lang.Integer getRequestedCount() { + return this.requestedCount; + } + + public java.lang.Integer effectiveCount; + public void setEffectiveCount(java.lang.Integer effectiveCount) { + this.effectiveCount = effectiveCount; + } + public java.lang.Integer getEffectiveCount() { + return this.effectiveCount; + } + + public java.lang.Boolean clamped; + public void setClamped(java.lang.Boolean clamped) { + this.clamped = clamped; + } + public java.lang.Boolean getClamped() { + return this.clamped; + } + + public java.util.List consumerKeys; + public void setConsumerKeys(java.util.List consumerKeys) { + this.consumerKeys = consumerKeys; + } + public java.util.List getConsumerKeys() { + return this.consumerKeys; + } + + public java.util.List unresolvedHandles; + public void setUnresolvedHandles(java.util.List unresolvedHandles) { + this.unresolvedHandles = unresolvedHandles; + } + public java.util.List getUnresolvedHandles() { + return this.unresolvedHandles; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerAction.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerAction.java new file mode 100644 index 00000000000..e2b00a1fcf0 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryPhysicalServerAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.QueryPhysicalServerResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.QueryPhysicalServerResult value = res.getResult(org.zstack.sdk.QueryPhysicalServerResult.class); + ret.value = value == null ? new org.zstack.sdk.QueryPhysicalServerResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/physical-servers"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentAction.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentAction.java new file mode 100644 index 00000000000..0603ecf0614 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryPhysicalServerResourceAssignmentAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult value = res.getResult(org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult.class); + ret.value = value == null ? new org.zstack.sdk.QueryPhysicalServerResourceAssignmentResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/physical-servers/resource-assignments"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentResult.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentResult.java new file mode 100644 index 00000000000..dbfa6e4dc32 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResourceAssignmentResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk; + + + +public class QueryPhysicalServerResourceAssignmentResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResult.java b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResult.java new file mode 100644 index 00000000000..33c83a32a37 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/QueryPhysicalServerResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk; + + + +public class QueryPhysicalServerResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerCpuIsolationAction.java b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerCpuIsolationAction.java new file mode 100644 index 00000000000..b521d6aea84 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerCpuIsolationAction.java @@ -0,0 +1,101 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class RefreshPhysicalServerCpuIsolationAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.RefreshPhysicalServerCpuIsolationResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String serverUuid; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.RefreshPhysicalServerCpuIsolationResult value = res.getResult(org.zstack.sdk.RefreshPhysicalServerCpuIsolationResult.class); + ret.value = value == null ? new org.zstack.sdk.RefreshPhysicalServerCpuIsolationResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/physical-servers/{serverUuid}/cpu-isolation/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "refreshPhysicalServerCpuIsolation"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerCpuIsolationResult.java b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerCpuIsolationResult.java new file mode 100644 index 00000000000..eda1b6c81d9 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/RefreshPhysicalServerCpuIsolationResult.java @@ -0,0 +1,7 @@ +package org.zstack.sdk; + + + +public class RefreshPhysicalServerCpuIsolationResult { + +} diff --git a/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerCpuAssignmentAction.java b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerCpuAssignmentAction.java new file mode 100644 index 00000000000..7a43ff7488b --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerCpuAssignmentAction.java @@ -0,0 +1,116 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UpdatePhysicalServerCpuAssignmentAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.UpdatePhysicalServerCpuAssignmentResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String serverUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String assignmentType; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String resourceType = "CPU"; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean enabled; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String selectionType; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String selectionValue; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.UpdatePhysicalServerCpuAssignmentResult value = res.getResult(org.zstack.sdk.UpdatePhysicalServerCpuAssignmentResult.class); + ret.value = value == null ? new org.zstack.sdk.UpdatePhysicalServerCpuAssignmentResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/physical-servers/{serverUuid}/cpu-assignments/{assignmentType}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "updatePhysicalServerCpuAssignment"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerCpuAssignmentResult.java b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerCpuAssignmentResult.java new file mode 100644 index 00000000000..3b732705eb6 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/UpdatePhysicalServerCpuAssignmentResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk; + +import org.zstack.sdk.PhysicalServerResourceAssignmentInventory; + +public class UpdatePhysicalServerCpuAssignmentResult { + public PhysicalServerResourceAssignmentInventory inventory; + public void setInventory(PhysicalServerResourceAssignmentInventory inventory) { + this.inventory = inventory; + } + public PhysicalServerResourceAssignmentInventory getInventory() { + return this.inventory; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index 6e222dd4959..60860dff23e 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -2203,28 +2203,49 @@ public void success(AddonInfo addonInfo) { if (param.isNewAdded()) { controller.onFirstAdditionConfigure(new NopeCompletion()); } + String oldAddonInfo = externalVO.getAddonInfo(); + boolean addonInfoChanged = addonInfo != null + && addonInfo.changed(oldAddonInfo); + Completion persist = new Completion(completion) { + @Override + public void success() { + String newAddonInfo = addonInfo == null ? null : addonInfo.serialize(); + String newUrl = StringUtils.isEmpty(self.getUrl()) + ? externalVO.getIdentity() + "://" + self.getUuid() + : self.getUrl(); + SQL.New(ExternalPrimaryStorageVO.class) + .eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) + .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfo) + .set(ExternalPrimaryStorageVO_.config, config) + .set(ExternalPrimaryStorageVO_.url, newUrl) + .update(); + externalVO.setAddonInfo(newAddonInfo); + externalVO.setConfig(config); + externalVO.setUrl(newUrl); + if (addonInfoChanged) { + ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = + new ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); + evtData.setUuid(self.getUuid()); + evtData.setAddonInfo(newAddonInfo); + evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + } + controller.setTrashExpireTime( + PrimaryStorageGlobalConfig.TRASH_EXPIRATION_TIME.value(Integer.class), + new NopeCompletion()); + pingHook(completion); + } - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.addonInfo, JSONObjectUtil.toJsonString(addonInfo)) - .set(ExternalPrimaryStorageVO_.config, config) - .set(ExternalPrimaryStorageVO_.url, StringUtils.isEmpty(self.getUrl()) ? - externalVO.getIdentity() + "://" + self.getUuid() : self.getUrl()) - .update(); - - if (addonInfo != null && addonInfo.changed(externalVO.getAddonInfo())) { - String newAddonInfo = addonInfo.serialize(); - SQL.New(ExternalPrimaryStorageVO.class).eq(ExternalPrimaryStorageVO_.uuid, self.getUuid()) - .set(ExternalPrimaryStorageVO_.addonInfo, newAddonInfo) - .update(); - externalVO.setAddonInfo(newAddonInfo); - // fire event when addonInfo changed - ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData evtData = new org.zstack.storage.addon.primary.ExternalPrimaryStorageCanonicalEvent.AddonInfoChangedData(); - evtData.setUuid(self.getUuid()); - evtData.setAddonInfo(newAddonInfo); - evtf.fire(ExternalPrimaryStorageCanonicalEvent.ADDON_INFO_CHANGED_PATH, evtData); + @Override + public void fail(ErrorCode errorCode) { + controller.syncAddonInfo(oldAddonInfo); + completion.fail(errorCode); + } + }; + if (addonInfoChanged) { + controller.beforePersistAddonInfo(addonInfo, persist); + } else { + persist.success(); } - controller.setTrashExpireTime(PrimaryStorageGlobalConfig.TRASH_EXPIRATION_TIME.value(Integer.class), new NopeCompletion()); - pingHook(completion); } @Override @@ -2250,9 +2271,28 @@ public void run(FlowTrigger trigger, Map data) { @Override public void success(PingResult ret) { if (ret.getAddonInfo() != null && ret.getAddonInfo().changed(externalVO.getAddonInfo())) { - updateAddonInfo(ret.getAddonInfo()); + String oldAddonInfo = externalVO.getAddonInfo(); + controller.beforePersistAddonInfo( + ret.getAddonInfo(), new Completion(trigger) { + @Override + public void success() { + updateAddonInfo(ret.getAddonInfo()); + completePing(ret); + } + + @Override + public void fail(ErrorCode errorCode) { + controller.syncAddonInfo(oldAddonInfo); + trigger.fail(errorCode); + } + }); + return; } + completePing(ret); + } + + private void completePing(PingResult ret) { if (!ret.isSuccess()) { trigger.fail(operr(ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10013, "ping external primary storage[%s] failed, %s", self.getUuid(), ret.getError())); return; diff --git a/test/src/test/resources/springConfigXml/Kvm.xml b/test/src/test/resources/springConfigXml/Kvm.xml index ec757841d12..be4326c1c12 100755 --- a/test/src/test/resources/springConfigXml/Kvm.xml +++ b/test/src/test/resources/springConfigXml/Kvm.xml @@ -109,6 +109,24 @@ + + + + + + + + + + + + + + + + + diff --git a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy index 74c3ac573dc..6d2bc4f0e9d 100644 --- a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy +++ b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy @@ -34608,6 +34608,64 @@ abstract class ApiHelper { } + def queryPhysicalServer(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryPhysicalServerAction.class) Closure c) { + def a = new org.zstack.sdk.QueryPhysicalServerAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def queryPhysicalServerResourceAssignment(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryPhysicalServerResourceAssignmentAction.class) Closure c) { + def a = new org.zstack.sdk.QueryPhysicalServerResourceAssignmentAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def queryPhysicalSwitch(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.QueryPhysicalSwitchAction.class) Closure c) { def a = new org.zstack.sdk.QueryPhysicalSwitchAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -38799,6 +38857,33 @@ abstract class ApiHelper { } + def refreshPhysicalServerCpuIsolation(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.RefreshPhysicalServerCpuIsolationAction.class) Closure c) { + def a = new org.zstack.sdk.RefreshPhysicalServerCpuIsolationAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def refreshPluginDrivers(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.RefreshPluginDriversAction.class) Closure c) { def a = new org.zstack.sdk.RefreshPluginDriversAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -47169,6 +47254,33 @@ abstract class ApiHelper { } + def updatePhysicalServerCpuAssignment(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdatePhysicalServerCpuAssignmentAction.class) Closure c) { + def a = new org.zstack.sdk.UpdatePhysicalServerCpuAssignmentAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def updatePluginSecretResourcePool(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.UpdatePluginSecretResourcePoolAction.class) Closure c) { def a = new org.zstack.sdk.UpdatePluginSecretResourcePoolAction() a.sessionId = Test.currentEnvSpec?.session?.uuid diff --git a/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy b/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy index a8e6812c992..7ef93c6871c 100755 --- a/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/SpringSpec.groovy @@ -114,6 +114,10 @@ class SpringSpec { include("cbd.xml") } + void physicalServer() { + include("physicalServer.xml") + } + void smp() { include("sharedMountPointPrimaryStorage.xml") }