diff --git a/compute/src/main/java/org/zstack/compute/legacy/ComputeLegacyGlobalProperty.java b/compute/src/main/java/org/zstack/compute/legacy/ComputeLegacyGlobalProperty.java new file mode 100644 index 00000000000..b1e4b91caf1 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/legacy/ComputeLegacyGlobalProperty.java @@ -0,0 +1,10 @@ +package org.zstack.compute.legacy; + +import org.zstack.core.GlobalProperty; +import org.zstack.core.GlobalPropertyDefinition; + +@GlobalPropertyDefinition +public class ComputeLegacyGlobalProperty { + @GlobalProperty(name="enable.nv.ram.type.volume", defaultValue = "false") + public static boolean enableNvRamTypeVolume; +} diff --git a/compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java b/compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java index 4a0d2ba9627..eee13650bb3 100644 --- a/compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java +++ b/compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java @@ -5,6 +5,7 @@ import org.zstack.header.vm.InstantiateNewCreatedVmInstanceMsg; import org.zstack.header.vm.VmCreationStrategy; import org.zstack.header.vm.VmNicSpec; +import org.zstack.header.vm.devices.VmDevicesSpec; import java.util.ArrayList; import java.util.List; @@ -31,6 +32,7 @@ public class InstantiateVmFromNewCreatedStruct { private List sshKeyPairUuids; private final List candidatePrimaryStorageUuidsForRootVolume = new ArrayList<>(); private final List candidatePrimaryStorageUuidsForDataVolume = new ArrayList<>(); + private VmDevicesSpec devicesSpec; public List getCandidatePrimaryStorageUuidsForRootVolume() { return candidatePrimaryStorageUuidsForRootVolume; @@ -64,6 +66,14 @@ public void setDiskAOs(List diskAOs) { this.diskAOs = diskAOs; } + public VmDevicesSpec getDevicesSpec() { + return devicesSpec; + } + + public void setDevicesSpec(VmDevicesSpec devicesSpec) { + this.devicesSpec = devicesSpec; + } + public List getRootVolumeSystemTags() { return rootVolumeSystemTags; } @@ -142,6 +152,7 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(InstantiateNewCreate struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex()); struct.setDisableL3Networks(msg.getDisableL3Networks()); struct.setDiskAOs(msg.getDiskAOs()); + struct.setDevicesSpec(msg.getDevicesSpec()); return struct; } @@ -161,6 +172,7 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(CreateVmInstanceMsg struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex()); struct.setDisableL3Networks(msg.getDisableL3Networks()); struct.setDiskAOs(msg.getDiskAOs()); + struct.setDevicesSpec(msg.getDevicesSpec()); return struct; } diff --git a/compute/src/main/java/org/zstack/compute/vm/VmCascadeExtension.java b/compute/src/main/java/org/zstack/compute/vm/VmCascadeExtension.java index 07247d60b91..63e25019a7f 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmCascadeExtension.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmCascadeExtension.java @@ -463,9 +463,11 @@ public void done(ErrorCodeList errorCodeList) { .flatMap(List::stream).map(VmCdRomInventory::getUuid) .collect(Collectors.toList()); dbf.removeByPrimaryKeys(cdRomUuids, VmCdRomVO.class); - dbf.removeByPrimaryKeys(vminvs.stream().map(p -> p.getInventory().getUuid()) - .collect(Collectors.toList()), - VmInstanceVO.class); + + List vmUuidList = vminvs.stream() + .map(vm -> vm.getInventory().getUuid()) + .collect(Collectors.toList()); + dbf.removeByPrimaryKeys(vmUuidList, VmInstanceVO.class); } completion.success(); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmDeleteVolumeFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmDeleteVolumeFlow.java index 8acb64f9bc3..c82f385f849 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmDeleteVolumeFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmDeleteVolumeFlow.java @@ -47,7 +47,9 @@ public void run(final FlowTrigger trigger, Map data) { final boolean deleteDataDisk = VmGlobalConfig.DELETE_DATA_VOLUME_ON_VM_DESTROY.value(Boolean.class); /* data volume must be detached anyway no matter if it is going to be deleted */ - if (spec.getVmInventory().getAllVolumes().size() > 1) { + boolean anyDataVolume = spec.getVmInventory().getAllVolumes().stream() + .anyMatch(arg -> VolumeType.Data.toString().equals(arg.getType())); + if (anyDataVolume) { detachDataVolumes(spec); } @@ -58,7 +60,11 @@ public void run(final FlowTrigger trigger, Map data) { return; } - List volumeTypes = Arrays.asList(VolumeType.Root.toString(), VolumeType.Memory.toString(), VolumeType.Cache.toString()); + List volumeTypes = Arrays.asList( + VolumeType.Root.toString(), + VolumeType.Memory.toString(), + VolumeType.Cache.toString() + ); List ctx = CollectionUtils.transformToList(spec.getVmInventory().getAllVolumes(), new Function() { @Override public VolumeDeletionStruct call(VolumeInventory arg) { diff --git a/compute/src/main/java/org/zstack/compute/vm/VmExpungeVmResourceCascadeFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmExpungeVmResourceCascadeFlow.java new file mode 100644 index 00000000000..9ec3bc73bbd --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/VmExpungeVmResourceCascadeFlow.java @@ -0,0 +1,83 @@ +package org.zstack.compute.vm; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.compute.vm.devices.VmTpmManager; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.core.cascade.CascadeFacade; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.header.core.Completion; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.vm.VmAfterExpungeExtensionPoint; +import org.zstack.header.vm.VmDeletionStruct; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.utils.CollectionUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Map; + +import static org.zstack.utils.CollectionDSL.list; + +/** + * Runs after volume/metadata expunge flows: fires {@link VmAfterExpungeExtensionPoint}, then + * {@link CascadeConstant#VM_INSTANCE_EXPUNGE_CODE} so TPM / VM host-file rows are removed before the VM row is deleted. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VmExpungeVmResourceCascadeFlow extends NoRollbackFlow { + private static final CLogger logger = Utils.getLogger(VmExpungeVmResourceCascadeFlow.class); + + /** + * Snapshot of TPM uuid before cascade deletes {@link org.zstack.header.tpm.entity.TpmVO}; + * read by {@link VmInstanceBase} expunge done handler for {@code detachTpmKeyProviderBestEffort}. + */ + public static final String EXPUNGE_CASCADE_TPM_UUID_KEY = + VmExpungeVmResourceCascadeFlow.class.getSimpleName() + ".tpmUuidBeforeCascade"; + + @Autowired + private CascadeFacade casf; + @Autowired + private PluginRegistry pluginRgty; + + @Override + public void run(FlowTrigger trigger, Map data) { + final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); + if (spec == null || spec.getVmInventory() == null) { + logger.warn("[VmExpungeVmResourceCascadeFlow] missing VmInstanceSpec, skip cascade"); + trigger.next(); + return; + } + + final VmInstanceInventory inv = spec.getVmInventory(); + + CollectionUtils.safeForEach(pluginRgty.getExtensionList(VmAfterExpungeExtensionPoint.class), + arg -> arg.vmAfterExpunge(inv)); + + final String tpmUuidBeforeCascade = VmTpmManager.findTpmUuidForVmOrNull(inv.getUuid()); + data.put(EXPUNGE_CASCADE_TPM_UUID_KEY, tpmUuidBeforeCascade); + + VmDeletionStruct expungeStruct = new VmDeletionStruct(); + expungeStruct.setInventory(inv); + expungeStruct.setDeletionPolicy(VmInstanceDeletionPolicy.Direct); + + casf.asyncCascade(CascadeConstant.VM_INSTANCE_EXPUNGE_CODE, VmInstanceVO.class.getSimpleName(), + list(expungeStruct), new Completion(trigger) { + @Override + public void success() { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java index 5f18eb7fd3f..ed522245552 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java @@ -117,6 +117,7 @@ public class VmGlobalConfig { @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "sync clock after vm resume") public static GlobalConfig VM_CLOCK_SYNC_AFTER_VM_RESUME = new GlobalConfig(CATEGORY, "vm.clock.sync.after.vm.resume"); + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) @GlobalConfigValidation(validValues = {"true", "false"}) public static GlobalConfig ENABLE_UEFI_SECURE_BOOT = new GlobalConfig(CATEGORY, "enable.uefi.secure.boot"); @@ -141,4 +142,13 @@ public class VmGlobalConfig { @GlobalConfigValidation(validValues = {"true", "false", "auto"}) @GlobalConfigDef(defaultValue = "false", type = String.class, description = "generate config required for vhost primary storage") public static GlobalConfig GENERATE_CONFIG_VHOST_REQUIRED = new GlobalConfig(CATEGORY, "generate.config.vhost.required"); + + @GlobalConfigDef(defaultValue = "true", type = Boolean.class, description = "whether reset TPM state after VM clone") + @GlobalConfigValidation(validValues = {"true", "false"}) + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) + public static GlobalConfig RESET_TPM_AFTER_VM_CLONE = new GlobalConfig(CATEGORY, "reset.tpm.after.vm.clone"); + + @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "allowed TPM VM start without KMS") + @GlobalConfigValidation(validValues = {"true", "false"}) + public static GlobalConfig ALLOWED_TPM_VM_WITHOUT_KMS = new GlobalConfig(CATEGORY, "allowed.tpm.vm.without.kms"); } diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java index 0e42d53cc71..e61d9489c79 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java @@ -1094,7 +1094,8 @@ private void validateRootDiskOffering(ImageMediaType imgFormat, APICreateVmInsta } if (msg.getRootDiskSize() <= 0) { - throw new ApiMessageInterceptionException(operr(ORG_ZSTACK_COMPUTE_VM_10182, "Unexpected root disk settings")); + throw new ApiMessageInterceptionException(operr(ORG_ZSTACK_COMPUTE_VM_10182, + "Unexpected root disk settings: DiskAO[0].size is mandatory when image format is ISO")); } } } diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index e5065e0f06a..098b14d3140 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -7,6 +7,8 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.transaction.annotation.Transactional; import org.zstack.compute.allocator.HostAllocatorManager; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend; +import org.zstack.compute.vm.devices.VmTpmManager; import org.zstack.core.Platform; import org.zstack.core.asyncbatch.While; import org.zstack.core.cascade.CascadeConstant; @@ -145,11 +147,26 @@ public class VmInstanceBase extends AbstractVmInstance { private TagManager tagMgr; @Autowired private VmInstanceDeviceManager vidm; + @Autowired + private TpmEncryptedResourceKeyBackend tpmKeyBackend; protected VmInstanceVO self; protected VmInstanceVO originalCopy; protected String syncThreadName; + private void detachTpmKeyProviderBestEffort(String tpmUuid) { + if (tpmUuid == null) { + return; + } + try { + tpmKeyBackend.detachKeyProviderFromTpm(tpmUuid); + } catch (Throwable t) { + logger.warn(String.format( + "failed to detach key provider from TPM[uuid:%s]: %s", + tpmUuid, t.getMessage()), t); + } + } + protected void checkState(final String hostUuid, final NoErrorCompletion completion) { CheckVmStateOnHypervisorMsg msg = new CheckVmStateOnHypervisorMsg(); msg.setVmInstanceUuids(list(self.getUuid())); @@ -1439,21 +1456,9 @@ private void expunge(Message msg, final Completion completion) { chain.done(new FlowDoneHandler(completion) { @Override public void handle(Map data) { - CollectionUtils.safeForEach(pluginRgty.getExtensionList(VmAfterExpungeExtensionPoint.class), - arg -> arg.vmAfterExpunge(inv)); - - callVmJustBeforeDeleteFromDbExtensionPoint(); - - dbf.reload(self); - dbf.removeCollection(self.getVmNics(), VmNicVO.class); - dbf.removeCollection(self.getVmCdRoms(), VmCdRomVO.class); - dbf.remove(self); - logger.debug(String.format("successfully expunged the vm[uuid:%s]", self.getUuid())); - dbf.eoCleanup(VmInstanceVO.class, self.getUuid()); - if (inv.getRootVolumeUuid() != null) { - dbf.eoCleanup(VolumeVO.class, inv.getRootVolumeUuid()); - } - completion.success(); + final String tpmUuidForEncryptedKeyRef = + (String) data.get(VmExpungeVmResourceCascadeFlow.EXPUNGE_CASCADE_TPM_UUID_KEY); + finishExpungeAfterVmResourceCascade(inv, tpmUuidForEncryptedKeyRef, completion); } }).error(new FlowErrorHandler(completion) { @Override @@ -1463,6 +1468,23 @@ public void handle(final ErrorCode errCode, Map data) { }).start(); } + private void finishExpungeAfterVmResourceCascade(VmInstanceInventory inv, String tpmUuidForEncryptedKeyRef, + Completion completion) { + callVmJustBeforeDeleteFromDbExtensionPoint(); + + self = dbf.reload(self); + dbf.removeCollection(self.getVmNics(), VmNicVO.class); + dbf.removeCollection(self.getVmCdRoms(), VmCdRomVO.class); + dbf.remove(self); + logger.debug(String.format("successfully expunged the vm[uuid:%s]", self.getUuid())); + dbf.eoCleanup(VmInstanceVO.class, self.getUuid()); + if (inv.getRootVolumeUuid() != null) { + dbf.eoCleanup(VolumeVO.class, inv.getRootVolumeUuid()); + } + detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef); + completion.success(); + } + private void handle(final VmCheckOwnStateMsg msg) { thdf.chainSubmit(new ChainTask(msg) { @Override @@ -2803,12 +2825,15 @@ public void success() { if (self.getState() != VmInstanceState.Destroyed) { changeVmStateInDb(VmInstanceStateEvent.destroyed); } + final String tpmUuidForEncryptedKeyRef = VmTpmManager.findTpmUuidForVmOrNull(self.getUuid()); callVmJustBeforeDeleteFromDbExtensionPoint(); dbf.removeCollection(self.getVmCdRoms(), VmCdRomVO.class); dbf.remove(getSelf()); dbf.eoCleanup(VmInstanceVO.class, self.getUuid()); + detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef); } else if (deletionPolicy == VmInstanceDeletionPolicy.DBOnly || deletionPolicy == VmInstanceDeletionPolicy.KeepVolume) { String accountUuid = acntMgr.getOwnerAccountUuidOfResource(inv.getUuid()); + final String tpmUuidForEncryptedKeyRef = VmTpmManager.findTpmUuidForVmOrNull(self.getUuid()); new SQLBatch() { @Override protected void scripts() { @@ -2822,6 +2847,7 @@ protected void scripts() { sql(VmInstanceVO.class).eq(VmInstanceVO_.uuid, self.getUuid()).hardDelete(); } }.execute(); + detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef); callVmJustAfterDeleteFromDbExtensionPoint(inv, accountUuid); } else if (deletionPolicy == VmInstanceDeletionPolicy.Delay) { changeVmStateInDb(VmInstanceStateEvent.destroyed); @@ -7767,6 +7793,7 @@ public DiskOfferingVO call(DiskOfferingVO arg) { } spec.setDiskAOs(struct.getDiskAOs()); + spec.setDevicesSpec(struct.getDevicesSpec()); List cdRomSpecs = buildVmCdRomSpecsForNewCreated(spec); spec.setCdRomSpecs(cdRomSpecs); @@ -7952,6 +7979,7 @@ public void handle(final Map data) { }).error(new FlowErrorHandler(completion) { @Override public void handle(final ErrorCode errCode, Map data) { + final String tpmUuidForEncryptedKeyRef = VmTpmManager.findTpmUuidForVmOrNull(self.getUuid()); extEmitter.failedToStartNewCreatedVm(VmInstanceInventory.valueOf(self), errCode); dbf.remove(self); // clean up EO, otherwise API-retry may cause conflict if @@ -7962,6 +7990,7 @@ public void handle(final ErrorCode errCode, Map data) { logger.warn(e.getMessage()); } + detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef); completion.fail(operr(ORG_ZSTACK_COMPUTE_VM_10289, errCode, errCode.getDetails())); } }).start(); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java index 321c7b3325d..538193ae362 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java @@ -924,7 +924,7 @@ private void handle(APICreateVmNicMsg msg) { FlowChain flowChain = FlowChainBuilder.newSimpleFlowChain(); flowChain.setName(String.format("create-nic-on-l3-network-%s", msg.getL3NetworkUuid())); flowChain.then(new NoRollbackFlow() { - String __name__ = "create-nic-and-presist-to-db"; + String __name__ = "create-nic-and-persist-to-db"; @Override public void run(FlowTrigger trigger, Map data) { @@ -1226,6 +1226,28 @@ public void setup() { attachOtherDisk = !otherDisks.isEmpty(); } + flow(new Flow() { + String __name__ = "call-after-persist-vm-extensions"; + List done = new ArrayList<>(); + + @Override + public void run(FlowTrigger trigger, Map data) { + for (VmInstanceCreateExtensionPoint extension : pluginRgty.getExtensionList(VmInstanceCreateExtensionPoint.class)) { + done.add(extension); + extension.afterPersistVmInstanceVO(finalVo, msg); + } + trigger.next(); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + Collections.reverse(done); + CollectionUtils.safeForEach(done, + extension -> extension.afterRollbackPersistVmInstanceVO(finalVo, msg)); + trigger.rollback(); + } + }); + flow(new Flow() { List errorCodes = new ArrayList<>(); String __name__ = String.format("instantiate-systemTag-for-vm-%s", finalVo.getUuid()); @@ -1323,6 +1345,7 @@ public void run(FlowTrigger trigger, Map data) { smsg.setDataVolumeSystemTags(msg.getDataVolumeSystemTags()); smsg.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex()); smsg.setDiskAOs(msg.getDiskAOs()); + smsg.setDevicesSpec(msg.getDevicesSpec()); bus.makeTargetServiceIdByResourceUuid(smsg, VmInstanceConstant.SERVICE_ID, finalVo.getUuid()); bus.send(smsg, new CloudBusCallBack(smsg) { @Override @@ -1364,7 +1387,7 @@ public void rollback(FlowRollback chain, Map data) { } DestroyVmInstanceMsg dmsg = new DestroyVmInstanceMsg(); dmsg.setVmInstanceUuid(finalVo.getUuid()); - dmsg.setDeletionPolicy(VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy.Direct); + dmsg.setDeletionPolicy(VmInstanceDeletionPolicy.Direct); bus.makeTargetServiceIdByResourceUuid(dmsg, VmInstanceConstant.SERVICE_ID, finalVo.getUuid()); bus.send(dmsg, new CloudBusCallBack(null) { @Override diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java index f0bdeded2af..208e265c968 100644 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java @@ -65,7 +65,7 @@ public static CreateVmInstanceMsg fromAPICreateVmInstanceMsg(APICreateVmInstance cmsg.setStrategy(msg.getStrategy()); cmsg.setDiskAOs(msg.getDiskAOs()); - + cmsg.setDevicesSpec(msg.getDevicesSpec()); if (CollectionUtils.isNotEmpty(msg.getDataDiskOfferingUuids()) || CollectionUtils.isNotEmpty(msg.getDataDiskSizes())) { cmsg.setPrimaryStorageUuidForDataVolume(getPSUuidForDataVolume(msg.getSystemTags())); } diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/DummyEncryptedResourceKeyManager.java b/compute/src/main/java/org/zstack/compute/vm/devices/DummyEncryptedResourceKeyManager.java new file mode 100644 index 00000000000..6563fa16d7f --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/DummyEncryptedResourceKeyManager.java @@ -0,0 +1,37 @@ +package org.zstack.compute.vm.devices; + +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.core.Completion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class DummyEncryptedResourceKeyManager implements EncryptedResourceKeyManager { + private static final CLogger logger = Utils.getLogger(DummyEncryptedResourceKeyManager.class); + + @Override + public void getOrCreateKey(GetOrCreateResourceKeyContext ctx, + ReturnValueCompletion completion) { + logger.warn(String.format("crypto module not installed, cannot create resource key for %s[uuid:%s]", + ctx.getResourceType(), ctx.getResourceUuid())); + completion.fail(operr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "crypto module is not installed, cannot manage resource encryption keys")); + } + + @Override + public ResourceKeyResult getKey(GetOrCreateResourceKeyContext ctx) { + logger.warn(String.format("crypto module not installed, cannot get resource key for %s[uuid:%s]", + ctx.getResourceType(), ctx.getResourceUuid())); + throw new OperationFailureException(operr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "crypto module is not installed, cannot manage resource encryption keys")); + } + + @Override + public void rollbackCreatedKey(ResourceKeyResult result, Completion completion) { + completion.success(); + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/DummyTpmEncryptedResourceKeyBackend.java b/compute/src/main/java/org/zstack/compute/vm/devices/DummyTpmEncryptedResourceKeyBackend.java new file mode 100644 index 00000000000..4ee23a2a02d --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/DummyTpmEncryptedResourceKeyBackend.java @@ -0,0 +1,88 @@ +package org.zstack.compute.vm.devices; + +import org.zstack.header.core.Completion; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +public class DummyTpmEncryptedResourceKeyBackend implements TpmEncryptedResourceKeyBackend { + private static final CLogger logger = Utils.getLogger(DummyTpmEncryptedResourceKeyBackend.class); + + @Override + public void attachKeyProviderToTpm(String tpmUuid, String keyProviderUuid) { + logger.debug("ignore attach key provider to TPM request for TPM uuid " + tpmUuid + + " and key provider uuid " + keyProviderUuid); + } + + @Override + public void detachKeyProviderFromTpm(String tpmUuid) { + logger.debug("ignore detach key provider from TPM request for TPM uuid " + tpmUuid); + } + + @Override + public String findKeyProviderUuidByTpm(String tpmUuid) { + return null; + } + + @Override + public String findKeyProviderUuidByName(String providerName) { + return null; + } + + @Override + public String findKeyProviderNameByTpm(String tpmUuid) { + return null; + } + + @Override + public Integer findKeyVersionByTpm(String tpmUuid) { + return null; + } + + @Override + public String defaultKeyProviderUuid() { + return null; + } + + @Override + public int applyKeyProviderWithKek(String tpmUuid, String providerUuid) { + return 0; + } + + @Override + public boolean checkTpmKeyProviderAttached(String tpmUuid) { + return false; + } + + @Override + public void cloneEncryptedResourceKey(CloneEncryptedResourceKeyContext context, Completion completion) { + // do nothing + logger.debug("ignore clone encrypted resource key request for TPM uuid " + + context.srcTpmUuid + " -> " + context.dstTpmUuid); + completion.success(); + } + + @Override + public void backupEncryptedResourceKey(BackupEncryptedResourceKeyContext context) { + // do nothing + logger.debug("ignore backup encrypted resource key request for src resource: " + + context.srcResourceUuid + " -> dest resource: " + context.dstResourceUuid); + } + + @Override + public void restoreEncryptedResourceKey(RestoreEncryptedResourceKeyContext context) { + // do nothing + logger.debug("ignore restore encrypted resource key request for src resource: " + + context.srcResourceUuid + " -> dest resource: " + context.dstResourceUuid); + } + + @Override + public void cleanEncryptedResourceKey(String vmHostBackupFileUuid) { + // do nothing + logger.debug("ignore cleanup encrypted resource key request for VmHostBackupFileVO: " + vmHostBackupFileUuid); + } + + @Override + public void cleanTpmKeyBackupEncryptedResourceKey(String tpmKeyBackupUuid) { + logger.debug("ignore cleanup encrypted resource key request for TpmKeyBackupVO: " + tpmKeyBackupUuid); + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/DummyVmHostFileManager.java b/compute/src/main/java/org/zstack/compute/vm/devices/DummyVmHostFileManager.java new file mode 100644 index 00000000000..a832714c35a --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/DummyVmHostFileManager.java @@ -0,0 +1,20 @@ +package org.zstack.compute.vm.devices; + +import org.zstack.core.db.SQL; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileManager; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +public class DummyVmHostFileManager implements VmHostFileManager { + private static final CLogger logger = Utils.getLogger(DummyVmHostFileManager.class); + + @Override + public void cleanVmHostBackupFile(String resourceUuid) { + logger.debug(String.format("clean VmHostBackupFileVO[resourceUuid=%s]", resourceUuid)); + SQL.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, resourceUuid) + .delete(); + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/TpmApiInterceptor.java b/compute/src/main/java/org/zstack/compute/vm/devices/TpmApiInterceptor.java new file mode 100644 index 00000000000..1acdcc5070f --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/TpmApiInterceptor.java @@ -0,0 +1,115 @@ +package org.zstack.compute.vm.devices; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.db.Q; +import org.zstack.header.apimediator.ApiMessageInterceptionException; +import org.zstack.header.apimediator.ApiMessageInterceptor; +import org.zstack.header.apimediator.StopRoutingException; +import org.zstack.header.errorcode.SysErrors; +import org.zstack.header.message.APIMessage; +import org.zstack.header.tpm.api.*; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import static org.zstack.core.Platform.err; +import static org.zstack.header.tpm.TpmConstants.*; +import static org.zstack.header.tpm.TpmErrors.*; +import static org.zstack.header.vm.VmInstanceConstant.KVM_HYPERVISOR_TYPE; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class TpmApiInterceptor implements ApiMessageInterceptor { + private static final CLogger logger = Utils.getLogger(TpmApiInterceptor.class); + + @Autowired + private CloudBus bus; + + @Override + public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionException { + if (msg instanceof APIGetTpmCapabilityMsg) { + validate((APIGetTpmCapabilityMsg) msg); + } else if (msg instanceof APIAddTpmMsg) { + validate((APIAddTpmMsg) msg); + } else if (msg instanceof APIRemoveTpmMsg) { + validate((APIRemoveTpmMsg) msg); + } else if (msg instanceof APIUpdateTpmMsg) { + validate((APIUpdateTpmMsg) msg); + } + + return msg; + } + + private void validate(APIGetTpmCapabilityMsg msg) { + makeSureVmInstanceIsKvmType(msg.getVmInstanceUuid()); + } + + private void validate(APIAddTpmMsg msg) { + makeSureVmInstanceIsKvmType(msg.getVmInstanceUuid()); + + boolean tpmExists = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, msg.getVmInstanceUuid()) + .isExists(); + if (tpmExists) { + throw new ApiMessageInterceptionException(err(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + TPM_ALREADY_EXISTS, "tpm device already exists")); + } + + boolean vmInSupportState = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, msg.getVmInstanceUuid()) + .in(VmInstanceVO_.state, SUPPORT_VM_STATES_FOR_TPM_OPERATION) + .isExists(); + if (!vmInSupportState) { + throw new ApiMessageInterceptionException(err(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, VM_STATE_ERROR, + "The current VM state does not support adding TPM operations") + .withOpaque("support.vm.state", SUPPORT_VM_STATES_FOR_TPM_OPERATION)); + } + + if (msg.getResourceUuid() == null) { + msg.setResourceUuid(Platform.getUuid()); + } + } + + private void validate(APIRemoveTpmMsg msg) { + makeSureVmInstanceIsKvmType(msg.getVmInstanceUuid()); + + boolean tpmExists = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, msg.getVmInstanceUuid()) + .isExists(); + if (!tpmExists) { + APIRemoveTpmEvent evt = new APIRemoveTpmEvent(msg.getId()); + bus.publish(evt); + throw new StopRoutingException(); + } + + boolean vmInSupportState = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, msg.getVmInstanceUuid()) + .in(VmInstanceVO_.state, SUPPORT_VM_STATES_FOR_TPM_OPERATION) + .isExists(); + if (!vmInSupportState) { + throw new ApiMessageInterceptionException(err(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, VM_STATE_ERROR, + "The current VM state does not support removing TPM operations") + .withOpaque("support.vm.state", SUPPORT_VM_STATES_FOR_TPM_OPERATION)); + } + } + + private void validate(APIUpdateTpmMsg msg) { + makeSureVmInstanceIsKvmType(msg.getVmInstanceUuid()); + bus.makeTargetServiceIdByResourceUuid(msg, SERVICE_ID, msg.getTpmUuid()); + } + + private void makeSureVmInstanceIsKvmType(String vmInstanceUuid) { + String hypervisorType = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hypervisorType) + .eq(VmInstanceVO_.uuid, vmInstanceUuid) + .findValue(); + if (!KVM_HYPERVISOR_TYPE.equals(hypervisorType)) { + throw new ApiMessageInterceptionException(err(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + SysErrors.NOT_SUPPORTED, "only allowed for kvm type VM instance")); + } + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/TpmCascadeExtension.java b/compute/src/main/java/org/zstack/compute/vm/devices/TpmCascadeExtension.java new file mode 100644 index 00000000000..a0b8e48a524 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/TpmCascadeExtension.java @@ -0,0 +1,178 @@ +package org.zstack.compute.vm.devices; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.message.MessageReply; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.tpm.message.TpmDeletionMsg; +import org.zstack.header.vm.VmDeletionStruct; +import org.zstack.header.vm.VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.utils.CollectionUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +import static org.zstack.core.Platform.operr; +import static org.zstack.header.tpm.TpmConstants.SERVICE_ID; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.CollectionUtils.transformAndRemoveNull; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class TpmCascadeExtension extends AbstractAsyncCascadeExtension { + private static final CLogger logger = Utils.getLogger(TpmCascadeExtension.class); + + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + + private static final String NAME = TpmVO.class.getSimpleName(); + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (action.isActionCode(CascadeConstant.DELETION_CHECK_CODE)) { + handleDeletionCheck(action, completion); + } else if (action.isActionCode(CascadeConstant.DELETION_DELETE_CODE, CascadeConstant.DELETION_FORCE_DELETE_CODE) + || action.isActionCode(CascadeConstant.VM_INSTANCE_EXPUNGE_CODE)) { + if (shouldDeferVmAssociatedDeletion(action)) { + completion.success(); + return; + } + handleDeletion(action, completion); + } else if (action.isActionCode(CascadeConstant.DELETION_CLEANUP_CODE)) { + handleDeletionCleanup(action, completion); + } else { + completion.success(); + } + } + + /** + * When VM deletion policy keeps the VM row (recycle bin), defer TPM removal until {@link CascadeConstant#VM_INSTANCE_EXPUNGE_CODE}. + */ + private boolean shouldDeferVmAssociatedDeletion(CascadeAction action) { + if (CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode())) { + return false; + } + if (!VmInstanceVO.class.getSimpleName().equals(action.getParentIssuer())) { + return false; + } + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return false; + } + for (Object o : (List) raw) { + if (!(o instanceof VmDeletionStruct)) { + continue; + } + VmDeletionStruct s = (VmDeletionStruct) o; + VmInstanceDeletionPolicy p = s.getDeletionPolicy(); + if (p == VmInstanceDeletionPolicy.Delay || p == VmInstanceDeletionPolicy.Never) { + return true; + } + } + return false; + } + + private List tpmFromAction(CascadeAction action) { + if (VmInstanceVO.class.getSimpleName().equals(action.getParentIssuer())) { + List vmDeletionStructs = action.getParentIssuerContext(); + List vmUuidList = transformAndRemoveNull(vmDeletionStructs, it -> it.getInventory().getUuid()); + + if (vmUuidList.isEmpty()) { + return null; + } + return Q.New(TpmVO.class) + .in(TpmVO_.vmInstanceUuid, vmUuidList) + .list(); + } else if (NAME.equals(action.getParentIssuer())) { + return action.getParentIssuerContext(); + } + + return null; + } + + private void handleDeletionCheck(CascadeAction action, Completion completion) { + completion.success(); + } + + private void handleDeletion(CascadeAction action, Completion completion) { + final List tpmList = tpmFromAction(action); + if (CollectionUtils.isEmpty(tpmList)) { + completion.success(); + return; + } + + new While<>(tpmList).each((tpm, whileCompletion) -> { + TpmDeletionMsg msg = new TpmDeletionMsg(); + msg.setTpmUuid(tpm.getUuid()); + msg.setVmInstanceUuid(tpm.getVmInstanceUuid()); + // delete TPM in cascade must skip VM state checking -> force should always BE TRUE + msg.setForceDelete(true); + bus.makeTargetServiceIdByResourceUuid(msg, SERVICE_ID, msg.getTpmUuid()); + bus.send(msg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + logger.debug(String.format("deleted Tpm[uuid:%s] from VM[uuid:%s]", + tpm.getUuid(), tpm.getVmInstanceUuid())); + whileCompletion.done(); + } else { + whileCompletion.addError(reply.getError()); + whileCompletion.allDone(); + } + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(operr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "failed to delete Tpm from VM[uuid:%s]", tpmList.get(0).getVmInstanceUuid()) + .causedBy(errorCodeList.getCauses())); + return; + } + completion.success(); + } + }); + } + + private void handleDeletionCleanup(CascadeAction action, Completion completion) { + completion.success(); + } + + @Override + public List getEdgeNames() { + return list(VmInstanceVO.class.getSimpleName()); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + if (CascadeConstant.DELETION_CODES.contains(action.getActionCode()) + || CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode())) { + List ctx = tpmFromAction(action); + if (ctx != null) { + return action.copy().setParentIssuer(NAME).setParentIssuerContext(ctx); + } + } + + return null; + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/TpmEncryptedResourceKeyBackend.java b/compute/src/main/java/org/zstack/compute/vm/devices/TpmEncryptedResourceKeyBackend.java new file mode 100644 index 00000000000..0db19ee4525 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/TpmEncryptedResourceKeyBackend.java @@ -0,0 +1,105 @@ +package org.zstack.compute.vm.devices; + +import org.zstack.header.core.Completion; + +/** + * Responsible for handling the replication or reset of encryption resource keys + * and other tasks in VM TPM cloning scenarios. + */ +public interface TpmEncryptedResourceKeyBackend { + + /** + * Build relationship from {@link org.zstack.header.tpm.entity.TpmVO} to EncryptedResourceKeyRefVO + * Non-async call. + */ + void attachKeyProviderToTpm(String tpmUuid, String keyProviderUuid); + + /** + * Clean relationship from {@link org.zstack.header.tpm.entity.TpmVO} to EncryptedResourceKeyRefVO + * Non-async call. + */ + void detachKeyProviderFromTpm(String tpmUuid); + + /** + * maybe null (when crypto module is not installed) + */ + String findKeyProviderUuidByTpm(String tpmUuid); + + /** + * maybe null (when crypto module is not installed) + */ + String findKeyProviderUuidByName(String providerName); + + /** + * maybe null (when crypto module is not installed) + */ + String findKeyProviderNameByTpm(String tpmUuid); + + /** + * maybe null (when crypto module is not installed) + */ + Integer findKeyVersionByTpm(String tpmUuid); + + /** + * maybe null (when crypto module is not installed) + */ + String defaultKeyProviderUuid(); + + /** + * Repair providerUuid on existing TPM key ref row where wrapped material already exists. + * @return updated rows + */ + int applyKeyProviderWithKek(String tpmUuid, String providerUuid); + + /** + * Whether an EncryptedResourceKeyRef row exists for this TPM. + */ + boolean checkTpmKeyProviderAttached(String tpmUuid); + + static class CloneEncryptedResourceKeyContext { + public String srcTpmUuid; + public String dstTpmUuid; + + /** + * Whether to reset (regenerate) the key on the target TPM. + *
    + *
  • {@code true}：Regenerate the key for the target TPM + * without inheriting the encrypted data from the source TPM.
    • + *
  • {@code false}：Copy the existing keys from the source TPM + * to the target TPM to ensure they remain consistent.
    • + *
+ */ + public boolean resetTpm; + } + + /** + * In a VM cloning scenario, copy or reset the encryption resource key + * from the source TPM to the target TPM. + */ + void cloneEncryptedResourceKey(CloneEncryptedResourceKeyContext context, Completion completion); + + static class BackupEncryptedResourceKeyContext { + public String srcResourceUuid; + public String dstResourceUuid; + } + + /** + * Copy the encryption resource key from the source TPM to {@link org.zstack.header.vm.additions.VmHostBackupFileVO} + * or other resources. + */ + void backupEncryptedResourceKey(BackupEncryptedResourceKeyContext context); + + static class RestoreEncryptedResourceKeyContext { + public String srcResourceUuid; + public String dstResourceUuid; + } + + void restoreEncryptedResourceKey(RestoreEncryptedResourceKeyContext context); + + void cleanEncryptedResourceKey(String vmHostBackupFileUuid); + + /** + * Remove encryption key material stored for a {@link org.zstack.header.tpm.entity.TpmKeyBackupVO}. + */ + void cleanTpmKeyBackupEncryptedResourceKey(String tpmKeyBackupUuid); +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/TpmMessageAutoCompleter.java b/compute/src/main/java/org/zstack/compute/vm/devices/TpmMessageAutoCompleter.java new file mode 100644 index 00000000000..af13333c7d8 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/TpmMessageAutoCompleter.java @@ -0,0 +1,80 @@ +package org.zstack.compute.vm.devices; + +import org.zstack.core.db.Q; +import org.zstack.header.apimediator.ApiMessageInterceptionException; +import org.zstack.header.apimediator.GlobalApiMessageInterceptor; +import org.zstack.header.message.APIDeleteMessage; +import org.zstack.header.message.APIMessage; +import org.zstack.header.tpm.api.APIGetTpmCapabilityMsg; +import org.zstack.header.tpm.api.APIRemoveTpmMsg; +import org.zstack.header.tpm.api.APIUpdateTpmMsg; +import org.zstack.header.tpm.api.TpmMessage; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; + +import java.util.List; + +import static org.zstack.core.Platform.argerr; +import static org.zstack.core.Platform.err; +import static org.zstack.header.tpm.TpmErrors.TPM_NOT_FOUND; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class TpmMessageAutoCompleter implements GlobalApiMessageInterceptor { + @Override + public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionException { + if (msg instanceof TpmMessage) { + validate((TpmMessage) msg); + } + return msg; + } + + @Override + @SuppressWarnings("rawtypes") + public List getMessageClassToIntercept() { + return list(APIGetTpmCapabilityMsg.class, APIRemoveTpmMsg.class, APIUpdateTpmMsg.class); + } + + @Override + public InterceptorPosition getPosition() { + return InterceptorPosition.FRONT; + } + + private void validate(TpmMessage msg) throws ApiMessageInterceptionException { + String tpmUuid = msg.getTpmUuid(); + String vmUuid = msg.getVmInstanceUuid(); + + if (tpmUuid == null && vmUuid == null) { + throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "tpmUuid and vmInstanceUuid cannot be null at the same time")); + } + + if (tpmUuid != null && vmUuid != null) { + boolean exists = Q.New(TpmVO.class) + .eq(TpmVO_.uuid, tpmUuid) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .isExists(); + if (!exists) { + throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "tpmUuid[%s] and vmInstanceUuid[%s] are not consistent", tpmUuid, vmUuid)); + } + } else if (vmUuid != null) { + tpmUuid = Q.New(TpmVO.class) + .select(TpmVO_.uuid) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .findValue(); + if (tpmUuid == null && (!(msg instanceof APIDeleteMessage))) { + throw new ApiMessageInterceptionException(err(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + TPM_NOT_FOUND, "tpm for vm[%s] does not exist", vmUuid)); + } else { + msg.setTpmUuid(tpmUuid); + } + } else { + vmUuid = Q.New(TpmVO.class) + .select(TpmVO_.vmInstanceUuid) + .eq(TpmVO_.uuid, tpmUuid) + .findValue(); + msg.setVmInstanceUuid(vmUuid); + } + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/VmHostBackupFileCascadeExtension.java b/compute/src/main/java/org/zstack/compute/vm/devices/VmHostBackupFileCascadeExtension.java new file mode 100644 index 00000000000..e316a429e19 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/VmHostBackupFileCascadeExtension.java @@ -0,0 +1,249 @@ +package org.zstack.compute.vm.devices; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.message.MessageReply; +import org.zstack.header.vm.VmDeletionStruct; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.utils.CollectionUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.CollectionUtils.transformAndRemoveNull; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class VmHostBackupFileCascadeExtension extends AbstractAsyncCascadeExtension { + private static final CLogger logger = Utils.getLogger(VmHostBackupFileCascadeExtension.class); + + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + + private static final String NAME = VmHostBackupFileVO.class.getSimpleName(); + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (action.isActionCode(CascadeConstant.DELETION_CHECK_CODE)) { + handleDeletionCheck(action, completion); + } else if (action.isActionCode(CascadeConstant.VM_INSTANCE_EXPUNGE_CODE)) { + handleDeletion(action, completion); + } else if (action.isActionCode(CascadeConstant.DELETION_DELETE_CODE, CascadeConstant.DELETION_FORCE_DELETE_CODE)) { + if (shouldDeferVmAssociatedDeletion(action)) { + completion.success(); + return; + } + handleDeletion(action, completion); + } else if (action.isActionCode(CascadeConstant.DELETION_CLEANUP_CODE)) { + handleDeletionCleanup(action, completion); + } else { + completion.success(); + } + } + + private boolean shouldDeferVmAssociatedDeletion(CascadeAction action) { + if (CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode())) { + return false; + } + if (!VmInstanceVO.class.getSimpleName().equals(action.getParentIssuer())) { + return false; + } + if (hasCreatedVmInDeletionContext(action)) { + logger.info(String.format( + "VmHostBackupFileCascadeExtension: skip deferring backup-file deletion for Created VM(s): %s; " + + "destroy uses DBOnly hard-delete without expunge, backup rows must be removed in this cascade", + formatCreatedVmUuidsFromContext(action))); + return false; + } + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return false; + } + for (Object o : (List) raw) { + if (!(o instanceof VmDeletionStruct)) { + continue; + } + VmDeletionStruct s = (VmDeletionStruct) o; + VmInstanceDeletionPolicy p = s.getDeletionPolicy(); + if (p == VmInstanceDeletionPolicy.Delay || p == VmInstanceDeletionPolicy.Never) { + return true; + } + } + return false; + } + + private boolean hasCreatedVmInDeletionContext(CascadeAction action) { + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return false; + } + for (Object o : (List) raw) { + if (!(o instanceof VmDeletionStruct)) { + continue; + } + VmInstanceInventory inv = ((VmDeletionStruct) o).getInventory(); + if (inv != null && VmInstanceState.Created.toString().equals(inv.getState())) { + return true; + } + } + return false; + } + + private String formatCreatedVmUuidsFromContext(CascadeAction action) { + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return "[]"; + } + return ((List) raw).stream() + .filter(VmDeletionStruct.class::isInstance) + .map(VmDeletionStruct.class::cast) + .map(VmDeletionStruct::getInventory) + .filter(inv -> inv != null && VmInstanceState.Created.toString().equals(inv.getState())) + .map(VmInstanceInventory::getUuid) + .collect(Collectors.joining(", ")); + } + + private List voFromAction(CascadeAction action) { + if (VmInstanceVO.class.getSimpleName().equals(action.getParentIssuer())) { + List vmDeletionStructs = action.getParentIssuerContext(); + List vmUuidList = transformAndRemoveNull(vmDeletionStructs, it -> it.getInventory().getUuid()); + + if (vmUuidList.isEmpty()) { + return null; + } + return Q.New(VmHostBackupFileVO.class) + .in(VmHostBackupFileVO_.resourceUuid, vmUuidList) + .list(); + } + // Note: VolumeSnapshotGroupVO has no cascade extension! + // skip: else if (VolumeSnapshotGroupVO.class.getSimpleName().equals(action.getParentIssuer())) + else if (NAME.equals(action.getParentIssuer())) { + return action.getParentIssuerContext(); + } + + return null; + } + + private void handleDeletionCheck(CascadeAction action, Completion completion) { + completion.success(); + } + + private void handleDeletion(CascadeAction action, Completion completion) { + final List voList = voFromAction(action); + if (CollectionUtils.isEmpty(voList)) { + completion.success(); + return; + } + + Set createdVmUuidsAsResource = findVmUuidsInCreatedState( + voList.stream().map(VmHostBackupFileVO::getResourceUuid).collect(Collectors.toSet())); + if (!createdVmUuidsAsResource.isEmpty()) { + logger.info(String.format( + "VmHostBackupFileCascadeExtension: deleting VmHostBackupFile row(s) tied to Created VM(s) %s with forceDelete=true " + + "(same as expunge path; avoids leftovers when VM row is hard-deleted)", + String.join(", ", createdVmUuidsAsResource))); + } + + new While<>(voList).each((vo, whileCompletion) -> { + VmHostBackupFileDeletionMsg msg = new VmHostBackupFileDeletionMsg(); + msg.setUuid(vo.getUuid()); + boolean force = action.isActionCode(CascadeConstant.DELETION_FORCE_DELETE_CODE) + || CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode()) + || createdVmUuidsAsResource.contains(vo.getResourceUuid()); + msg.setForceDelete(force); + bus.makeLocalServiceId(msg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(msg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + logger.debug(String.format("deleted VmHostBackupFile[uuid:%s] from resource[uuid:%s]", + vo.getUuid(), vo.getResourceUuid())); + whileCompletion.done(); + } else { + whileCompletion.addError(reply.getError()); + whileCompletion.allDone(); + } + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(operr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "failed to delete VmHostBackupFile from resource[uuid:%s]", voList.get(0).getResourceUuid()) + .causedBy(errorCodeList.getCauses())); + return; + } + completion.success(); + } + }); + } + + private Set findVmUuidsInCreatedState(Set candidateVmUuids) { + if (candidateVmUuids.isEmpty()) { + return new HashSet<>(); + } + return new HashSet<>(Q.New(VmInstanceVO.class) + .in(VmInstanceVO_.uuid, candidateVmUuids) + .eq(VmInstanceVO_.state, VmInstanceState.Created) + .select(VmInstanceVO_.uuid) + .listValues()); + } + + private void handleDeletionCleanup(CascadeAction action, Completion completion) { + completion.success(); + } + + @Override + public List getEdgeNames() { + return list( + VmInstanceVO.class.getSimpleName() + + // Note: VolumeSnapshotGroupVO has no cascade extension! + // skip: VolumeSnapshotGroupVO.class.getName() + ); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + if (CascadeConstant.DELETION_CODES.contains(action.getActionCode()) + || CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode())) { + List ctx = voFromAction(action); + if (ctx != null) { + return action.copy().setParentIssuer(NAME).setParentIssuerContext(ctx); + } + } + + return null; + } +} \ No newline at end of file diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/VmHostFileCascadeExtension.java b/compute/src/main/java/org/zstack/compute/vm/devices/VmHostFileCascadeExtension.java new file mode 100644 index 00000000000..d65f4062b8a --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/VmHostFileCascadeExtension.java @@ -0,0 +1,244 @@ +package org.zstack.compute.vm.devices; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cascade.AbstractAsyncCascadeExtension; +import org.zstack.core.cascade.CascadeAction; +import org.zstack.core.cascade.CascadeConstant; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.message.MessageReply; +import org.zstack.header.vm.VmDeletionStruct; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.additions.VmHostFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.utils.CollectionUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.CollectionUtils.transformAndRemoveNull; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class VmHostFileCascadeExtension extends AbstractAsyncCascadeExtension { + private static final CLogger logger = Utils.getLogger(VmHostFileCascadeExtension.class); + + @Autowired + private DatabaseFacade dbf; + @Autowired + private CloudBus bus; + + private static final String NAME = VmHostFileVO.class.getSimpleName(); + + @Override + public void asyncCascade(CascadeAction action, Completion completion) { + if (action.isActionCode(CascadeConstant.DELETION_CHECK_CODE)) { + handleDeletionCheck(action, completion); + } else if (action.isActionCode(CascadeConstant.VM_INSTANCE_EXPUNGE_CODE)) { + handleDeletion(action, completion); + } else if (action.isActionCode(CascadeConstant.DELETION_DELETE_CODE, CascadeConstant.DELETION_FORCE_DELETE_CODE)) { + if (shouldDeferVmAssociatedDeletion(action)) { + completion.success(); + return; + } + handleDeletion(action, completion); + } else if (action.isActionCode(CascadeConstant.DELETION_CLEANUP_CODE)) { + handleDeletionCleanup(action, completion); + } else { + completion.success(); + } + } + + private boolean shouldDeferVmAssociatedDeletion(CascadeAction action) { + if (CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode())) { + return false; + } + if (!VmInstanceVO.class.getSimpleName().equals(action.getParentIssuer())) { + return false; + } + if (hasCreatedVmInDeletionContext(action)) { + logger.info(String.format( + "VmHostFileCascadeExtension: skip deferring host-file deletion for Created VM(s): %s; " + + "destroy uses DBOnly hard-delete without expunge, host files must be removed in this cascade", + formatCreatedVmUuidsFromContext(action))); + return false; + } + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return false; + } + for (Object o : (List) raw) { + if (!(o instanceof VmDeletionStruct)) { + continue; + } + VmDeletionStruct s = (VmDeletionStruct) o; + VmInstanceDeletionPolicy p = s.getDeletionPolicy(); + if (p == VmInstanceDeletionPolicy.Delay || p == VmInstanceDeletionPolicy.Never) { + return true; + } + } + return false; + } + + private boolean hasCreatedVmInDeletionContext(CascadeAction action) { + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return false; + } + for (Object o : (List) raw) { + if (!(o instanceof VmDeletionStruct)) { + continue; + } + VmInstanceInventory inv = ((VmDeletionStruct) o).getInventory(); + if (inv != null && VmInstanceState.Created.toString().equals(inv.getState())) { + return true; + } + } + return false; + } + + private String formatCreatedVmUuidsFromContext(CascadeAction action) { + Object raw = action.getParentIssuerContext(); + if (!(raw instanceof List)) { + return "[]"; + } + return ((List) raw).stream() + .filter(VmDeletionStruct.class::isInstance) + .map(VmDeletionStruct.class::cast) + .map(VmDeletionStruct::getInventory) + .filter(inv -> inv != null && VmInstanceState.Created.toString().equals(inv.getState())) + .map(VmInstanceInventory::getUuid) + .collect(Collectors.joining(", ")); + } + + private List voFromAction(CascadeAction action) { + if (VmInstanceVO.class.getSimpleName().equals(action.getParentIssuer())) { + List vmDeletionStructs = action.getParentIssuerContext(); + List vmUuidList = transformAndRemoveNull(vmDeletionStructs, it -> it.getInventory().getUuid()); + + if (vmUuidList.isEmpty()) { + return null; + } + return Q.New(VmHostFileVO.class) + .in(VmHostFileVO_.vmInstanceUuid, vmUuidList) + .list(); + } else if (NAME.equals(action.getParentIssuer())) { + return action.getParentIssuerContext(); + } + + return null; + } + + private void handleDeletionCheck(CascadeAction action, Completion completion) { + completion.success(); + } + + private void handleDeletion(CascadeAction action, Completion completion) { + final List voList = voFromAction(action); + if (CollectionUtils.isEmpty(voList)) { + completion.success(); + return; + } + + Set createdVmUuids = findVmUuidsInCreatedState( + voList.stream().map(VmHostFileVO::getVmInstanceUuid).collect(Collectors.toSet())); + if (!createdVmUuids.isEmpty()) { + logger.info(String.format( + "VmHostFileCascadeExtension: deleting VmHostFile row(s) for Created VM(s) %s with forceDelete=true " + + "(same as expunge path; avoids leftovers when VM row is hard-deleted)", + String.join(", ", createdVmUuids))); + } + + new While<>(voList).each((vo, whileCompletion) -> { + VmHostFileDeletionMsg msg = new VmHostFileDeletionMsg(); + msg.setUuid(vo.getUuid()); + boolean force = action.isActionCode(CascadeConstant.DELETION_FORCE_DELETE_CODE) + || CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode()) + || createdVmUuids.contains(vo.getVmInstanceUuid()); + msg.setForceDelete(force); + bus.makeLocalServiceId(msg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(msg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + logger.debug(String.format("deleted VmHostFile[uuid:%s] from VM[uuid:%s]", + vo.getUuid(), vo.getVmInstanceUuid())); + whileCompletion.done(); + } else { + whileCompletion.addError(reply.getError()); + whileCompletion.allDone(); + } + } + }); + }).run(new WhileDoneCompletion(completion) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + completion.fail(operr(ORG_ZSTACK_COMPUTE_VM_DEVICES_10009, + "failed to delete VmHostFile from VM[uuid:%s]", voList.get(0).getVmInstanceUuid()) + .causedBy(errorCodeList.getCauses())); + return; + } + completion.success(); + } + }); + } + + private Set findVmUuidsInCreatedState(Set vmUuids) { + if (vmUuids.isEmpty()) { + return new HashSet<>(); + } + return new HashSet<>(Q.New(VmInstanceVO.class) + .in(VmInstanceVO_.uuid, vmUuids) + .eq(VmInstanceVO_.state, VmInstanceState.Created) + .select(VmInstanceVO_.uuid) + .listValues()); + } + + private void handleDeletionCleanup(CascadeAction action, Completion completion) { + completion.success(); + } + + @Override + public List getEdgeNames() { + // Note: host delete -> VmHostFileVO should not be delete: + // TPM/NvRam is too important and should not be deleted so easily. + // And hostUuid will not be updated -> host is not reachable. it is by design. + return list(VmInstanceVO.class.getSimpleName()); + } + + @Override + public String getCascadeResourceName() { + return NAME; + } + + @Override + public CascadeAction createActionForChildResource(CascadeAction action) { + if (CascadeConstant.DELETION_CODES.contains(action.getActionCode()) + || CascadeConstant.VM_INSTANCE_EXPUNGE_CODE.equals(action.getActionCode())) { + List ctx = voFromAction(action); + if (ctx != null) { + return action.copy().setParentIssuer(NAME).setParentIssuerContext(ctx); + } + } + + return null; + } +} \ No newline at end of file diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmExtensions.java b/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmExtensions.java new file mode 100644 index 00000000000..fe0535f27c6 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmExtensions.java @@ -0,0 +1,132 @@ +package org.zstack.compute.vm.devices; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.BuildVmSpecExtensionPoint; +import org.zstack.compute.vm.VmSystemTags; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQLBatch; +import org.zstack.header.tpm.entity.TpmSpec; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.vm.CreateVmInstanceMsg; +import org.zstack.header.vm.VmInstanceCreateExtensionPoint; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmMachineType; +import org.zstack.header.vm.devices.NvRamSpec; +import org.zstack.header.vm.devices.VmDevicesSpec; +import org.zstack.resourceconfig.ResourceConfig; +import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import static org.zstack.compute.vm.VmGlobalConfig.ENABLE_UEFI_SECURE_BOOT; + +public class VmTpmExtensions implements VmInstanceCreateExtensionPoint, + BuildVmSpecExtensionPoint { + private static final CLogger logger = Utils.getLogger(VmTpmExtensions.class); + + @Autowired + private VmTpmManager vmTpmManager; + @Autowired + private ResourceConfigFacade resourceConfigFacade; + @Autowired + private TpmEncryptedResourceKeyBackend resourceKeyBackend; + + @Override + public void preCreateVmInstance(CreateVmInstanceMsg msg) { + // do-nothing + } + + @Override + public void afterPersistVmInstanceVO(VmInstanceVO vo, CreateVmInstanceMsg msg) { + final VmDevicesSpec spec = msg.getDevicesSpec(); + if (spec == null || spec.getTpm() == null || !spec.getTpm().isEnable()) { + return; + } + + new SQLBatch() { + @Override + protected void scripts() { + final TpmVO tpm = vmTpmManager.persistTpmVO(null, vo.getUuid()); + final String keyProviderUuid = spec.getTpm().getKeyProviderUuid(); + if (keyProviderUuid != null) { + resourceKeyBackend.attachKeyProviderToTpm(tpm.getUuid(), keyProviderUuid); + } + } + }.execute(); + } + + @Override + public void afterRollbackPersistVmInstanceVO(VmInstanceVO vo, CreateVmInstanceMsg msg) { + String tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vo.getUuid()) + .select(TpmVO_.uuid) + .findValue(); + if (tpmUuid == null) { + return; + } + + new SQLBatch() { + @Override + protected void scripts() { + try { + resourceKeyBackend.detachKeyProviderFromTpm(tpmUuid); + } finally { + vmTpmManager.deleteTpmVO(tpmUuid); + } + } + }.execute(); + } + + @Override + public void afterBuildVmSpec(VmInstanceSpec spec) { + String vmUuid = spec.getVmInventory().getUuid(); + + String tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .select(TpmVO_.uuid) + .findValue(); + boolean needRegisterNvRam = tpmUuid != null; + if (!needRegisterNvRam) { + String bootMode = VmSystemTags.BOOT_MODE.getTokenByResourceUuid(vmUuid, VmSystemTags.BOOT_MODE_TOKEN); + if (vmTpmManager.isUefiBootMode(bootMode)) { + ResourceConfig resourceConfig = resourceConfigFacade.getResourceConfig(ENABLE_UEFI_SECURE_BOOT.getIdentity()); + needRegisterNvRam = resourceConfig.getResourceConfigValue(vmUuid, Boolean.class) == Boolean.TRUE; + } + } + + VmDevicesSpec devicesSpec = spec.getDevicesSpec(); + if (devicesSpec == null) { + devicesSpec = new VmDevicesSpec(); + spec.setDevicesSpec(devicesSpec); + } + + if (needRegisterNvRam) { + NvRamSpec nvRamSpec = devicesSpec.getNvRam(); + if (nvRamSpec == null) { + nvRamSpec = new NvRamSpec(); + devicesSpec.setNvRam(nvRamSpec); + } + + nvRamSpec.setNeedRegister(true); + } + + if (tpmUuid != null) { + TpmSpec tpmSpec = devicesSpec.getTpm(); + if (tpmSpec == null) { + tpmSpec = new TpmSpec(); + devicesSpec.setTpm(tpmSpec); + } + + tpmSpec.setEnable(true); + tpmSpec.setTpmUuid(tpmUuid); + } + + if (needRegisterNvRam && spec.getOsSpec().getMachineType() == null) { + spec.getOsSpec().setMachineType(VmMachineType.q35.toString()); + logger.debug(String.format( + "auto-set machineType to q35 for VM[uuid:%s] because NvRam/TPM registration is needed", vmUuid)); + } + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmManager.java b/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmManager.java new file mode 100644 index 00000000000..38e9cc2f990 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmManager.java @@ -0,0 +1,96 @@ +package org.zstack.compute.vm.devices; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.VmSystemTags; +import org.zstack.core.Platform; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.image.ImageBootMode; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.identity.AccountManager; +import org.zstack.resourceconfig.ResourceConfig; +import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +import static org.zstack.compute.vm.VmGlobalConfig.ENABLE_UEFI_SECURE_BOOT; +import static org.zstack.header.vm.additions.VmHostFileType.NvRam; +import static org.zstack.header.vm.additions.VmHostFileType.TpmState; +import static org.zstack.utils.CollectionDSL.list; + +public class VmTpmManager { + private static final CLogger logger = Utils.getLogger(VmTpmManager.class); + + @Autowired + private DatabaseFacade databaseFacade; + @Autowired + private ResourceConfigFacade resourceConfigFacade; + @Autowired + private AccountManager accountManager; + + public TpmVO persistTpmVO(String tpmUuid, String vmUuid) { + if (tpmUuid == null) { + tpmUuid = Platform.getUuid(); + } + TpmVO tpm = new TpmVO(); + tpm.setUuid(tpmUuid); + tpm.setResourceName("TPM-for-VM-" + vmUuid); + tpm.setVmInstanceUuid(vmUuid); + tpm.setAccountUuid(accountManager.getOwnerAccountUuidOfResource(vmUuid)); + databaseFacade.persistAndRefresh(tpm); + + logger.debug("Persisted TpmVO for VM " + vmUuid + " with uuid=" + tpm.getUuid()); + return tpm; + } + + public void deleteTpmVO(String tpmUuid) { + databaseFacade.removeByPrimaryKey(tpmUuid, TpmVO.class); + } + + public static String findTpmUuidForVmOrNull(String vmInstanceUuid) { + return Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vmInstanceUuid) + .select(TpmVO_.uuid) + .findValue(); + } + + /** + * @param bootMode boot mode, null is Legacy + */ + public static boolean isUefiBootMode(String bootMode) { + return Objects.equals(bootMode, ImageBootMode.UEFI.toString()) + || Objects.equals(bootMode, ImageBootMode.UEFI_WITH_CSM.toString()); + } + + public boolean needRegisterNvRam(String vmUuid) { + return needRegister(NvRam, vmUuid); + } + + public Set vmHostFileTypeNeedRegisterForVm(String vmUuid) { + String bootMode = VmSystemTags.BOOT_MODE.getTokenByResourceUuid(vmUuid, VmSystemTags.BOOT_MODE_TOKEN); + if (!isUefiBootMode(bootMode)) { + return Collections.emptySet(); + } + + boolean hasTpm = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .isExists(); + if (hasTpm) { + return new HashSet<>(list(NvRam, TpmState)); + } + ResourceConfig resourceConfig = resourceConfigFacade.getResourceConfig(ENABLE_UEFI_SECURE_BOOT.getIdentity()); + return resourceConfig.getResourceConfigValue(vmUuid, Boolean.class) == Boolean.TRUE ? + new HashSet<>(list(NvRam)) : Collections.emptySet(); + } + + public boolean needRegister(VmHostFileType type, String vmUuid) { + return vmHostFileTypeNeedRegisterForVm(vmUuid).contains(type); + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmRekeyAssociation.java b/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmRekeyAssociation.java new file mode 100644 index 00000000000..ae469a0a9b4 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/devices/VmTpmRekeyAssociation.java @@ -0,0 +1,35 @@ +package org.zstack.compute.vm.devices; + +import org.zstack.core.db.Q; +import org.zstack.header.keyprovider.KeyProviderRekeyAssociationExtensionPoint; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.utils.CollectionUtils; + +import java.util.Collections; +import java.util.List; + +public class VmTpmRekeyAssociation implements KeyProviderRekeyAssociationExtensionPoint { + @Override + public String getType() { + return VmInstanceVO.class.getSimpleName(); + } + + @Override + public String getAssociatedResourceType() { + return TpmVO.class.getSimpleName(); + } + + @Override + public List getAssociatedResourceUuids(List resourceUuids) { + if (CollectionUtils.isEmpty(resourceUuids)) { + return Collections.emptyList(); + } + + return Q.New(TpmVO.class) + .in(TpmVO_.vmInstanceUuid, resourceUuids) + .select(TpmVO_.uuid) + .listValues(); + } +} diff --git a/conf/db/upgrade/V5.5.38__schema.sql b/conf/db/upgrade/V5.5.38__schema.sql new file mode 100644 index 00000000000..f8564f6e2d3 --- /dev/null +++ b/conf/db/upgrade/V5.5.38__schema.sql @@ -0,0 +1,144 @@ +-- Feature: vTPM & Secure Boot | ZSPHER-1, ZSPHER-14 + +CREATE TABLE IF NOT EXISTS `zstack`.`TpmVO` ( + `uuid` char(32) NOT NULL UNIQUE, + `vmInstanceUuid` char(32) NOT NULL, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`), + CONSTRAINT `fkTpmVOVmInstanceVO` FOREIGN KEY (`vmInstanceUuid`) REFERENCES `VmInstanceEO` (`uuid`) ON UPDATE RESTRICT ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`VmHostFileVO` ( + `uuid` char(32) NOT NULL UNIQUE, + `vmInstanceUuid` char(32) NOT NULL, + `hostUuid` char(32) NOT NULL, + `type` varchar(64) NOT NULL COMMENT 'NvRam, TpmState', + `path` varchar(1024) NOT NULL COMMENT 'Absolute path of the file on the host', + `lastSyncReason` varchar(255) DEFAULT NULL COMMENT 'The reason for the last sync operation', + `changeDate` timestamp NULL DEFAULT NULL COMMENT 'Timestamp when file was reported changed, null after sync', + `lastSyncDate` timestamp NULL DEFAULT NULL COMMENT 'Timestamp of the last successful sync', + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`), + INDEX `idxVmHostFileVOVmInstanceUuid` (`vmInstanceUuid`), + INDEX `idxVmHostFileVOHostUuid` (`hostUuid`), + UNIQUE KEY `ukVmHostFileVO` (`vmInstanceUuid`, `hostUuid`, `type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`VmHostBackupFileVO` ( + `uuid` char(32) NOT NULL UNIQUE, + `resourceUuid` char(32) NOT NULL, + `type` varchar(64) NOT NULL COMMENT 'NvRam, TpmState', + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukVmHostBackupFileVO` (`resourceUuid`, `type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`VmHostFileContentVO` ( + `uuid` char(32) NOT NULL UNIQUE COMMENT 'VmHostFileVO.uuid or VmHostBackupFileVO.uuid', + `content` MEDIUMBLOB DEFAULT NULL, + `format` varchar(64) NOT NULL COMMENT 'Raw, TarballGzip', + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`), + CONSTRAINT `fkVmHostFileContentVOResourceVO` FOREIGN KEY (`uuid`) REFERENCES `ResourceVO` (`uuid`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Feature: KMS | ZSPHER-46, ZSPHER-60, ZSPHER-61, ZSPHER-62 + +CREATE TABLE IF NOT EXISTS `zstack`.`KeyProviderVO` ( + `uuid` varchar(32) NOT NULL UNIQUE, + `name` varchar(255) NOT NULL, + `description` varchar(2048) DEFAULT NULL, + `type` varchar(32) NOT NULL, + `connected` boolean NOT NULL DEFAULT FALSE, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukKeyProviderVOName` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`KmsVO` ( + `uuid` varchar(32) NOT NULL UNIQUE, + `endpoint` varchar(255) NOT NULL, + `port` int unsigned NOT NULL, + `kmipVersion` varchar(32) DEFAULT NULL, + `username` varchar(255) DEFAULT NULL, + `password` varchar(255) DEFAULT NULL, + `trustState` varchar(32) NOT NULL DEFAULT 'MUTUAL_UNTRUSTED', + `activeIdentityUuid` varchar(32) DEFAULT NULL, + `serverCertExpiredDate` timestamp NULL DEFAULT NULL, + `serverCertPem` text DEFAULT NULL, + PRIMARY KEY (`uuid`), + INDEX `idxKmsVOActiveIdentityUuid` (`activeIdentityUuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`KmsIdentityVO` ( + `uuid` varchar(32) NOT NULL UNIQUE, + `kmsUuid` varchar(32) NOT NULL, + `identityType` varchar(32) NOT NULL, + `clientCertPem` text DEFAULT NULL, + `clientKeyPem` text DEFAULT NULL, + `csrPem` text DEFAULT NULL, + `certExpiredDate` timestamp NULL DEFAULT NULL, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`), + UNIQUE KEY `ukKmsIdentityVOKmsUuidType` (`kmsUuid`, `identityType`), + INDEX `idxKmsIdentityVOKmsUuid` (`kmsUuid`), + CONSTRAINT `fkKmsIdentityVOKmsVO` FOREIGN KEY (`kmsUuid`) REFERENCES `KmsVO` (`uuid`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`NkpVO` ( + `uuid` varchar(32) NOT NULL UNIQUE, + `kdf` varchar(64) NOT NULL, + `saltPolicy` varchar(64) NOT NULL, + `backedUp` boolean NOT NULL DEFAULT FALSE, + `currentVersion` int unsigned DEFAULT NULL, + PRIMARY KEY (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`HostKeyIdentityVO` ( + `hostUuid` varchar(32) NOT NULL UNIQUE, + `publicKey` text NOT NULL, + `fingerprint` varchar(128) NOT NULL, + `verified` boolean NOT NULL DEFAULT FALSE, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`hostUuid`), + CONSTRAINT `fkHostKeyIdentityVOHostEO` FOREIGN KEY (`hostUuid`) REFERENCES `HostEO` (`uuid`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`TpmKeyBackupVO` ( + `uuid` char(32) NOT NULL UNIQUE, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `zstack`.`EncryptedResourceKeyRefVO` ( + `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, + `resourceType` varchar(255) NOT NULL, + `resourceUuid` varchar(32) NOT NULL, + `providerUuid` varchar(32) DEFAULT NULL, + `providerName` varchar(255) NOT NULL, + `keyVersion` int unsigned DEFAULT NULL, + `kekRef` varchar(255) DEFAULT NULL, + `wrappedDek` text NOT NULL, + `algorithm` varchar(64) DEFAULT NULL, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`id`), + INDEX `idxEncryptedResourceKeyRefVOResource` (`resourceType`, `resourceUuid`), + INDEX `idxEncryptedResourceKeyRefVOProviderUuid` (`providerUuid`), + INDEX `idxEncryptedResourceKeyRefVOProviderName` (`providerName`), + CONSTRAINT `fkEncryptedResourceKeyRefVOProviderUuid` FOREIGN KEY (`providerUuid`) REFERENCES `KeyProviderVO` (`uuid`) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DELETE FROM `EncryptedResourceKeyRefVO` + WHERE `resourceUuid` NOT IN (SELECT `uuid` FROM `ResourceVO`); +ALTER TABLE `EncryptedResourceKeyRefVO` + ADD CONSTRAINT `fkEncryptedResourceKeyRefResourceVO` FOREIGN KEY (`resourceUuid`) REFERENCES `ResourceVO`(`uuid`) + ON DELETE CASCADE; diff --git a/conf/errorCodes/sys.xml b/conf/errorCodes/sys.xml index 71cce02af39..9a0f4008b5a 100755 --- a/conf/errorCodes/sys.xml +++ b/conf/errorCodes/sys.xml @@ -90,5 +90,10 @@ 1017 A message or a operation cancelled + + + 1091 + Operation not supported + diff --git a/conf/errorCodes/tpm.xml b/conf/errorCodes/tpm.xml new file mode 100644 index 00000000000..d55f1a1308a --- /dev/null +++ b/conf/errorCodes/tpm.xml @@ -0,0 +1,23 @@ + + TPM + + + 1000 + General error + + + + 1701 + TPM already exists in this VM + + + + 1702 + TPM not found + + + + 1703 + The current VM state does not support this TPM operations + + diff --git a/conf/globalConfig/vm.xml b/conf/globalConfig/vm.xml index a7f835c82ee..9da4228ad91 100755 --- a/conf/globalConfig/vm.xml +++ b/conf/globalConfig/vm.xml @@ -289,7 +289,7 @@ enable.uefi.secure.boot enable uefi secure boot - true + false vm java.lang.Boolean diff --git a/conf/i18n/globalErrorCodeMapping/global-error-de-DE.json b/conf/i18n/globalErrorCodeMapping/global-error-de-DE.json index b3602b32e16..b29ab243858 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-de-DE.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-de-DE.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "Virtuelle Maschine mit UUID: %s nicht gefunden", "ORG_ZSTACK_VPC_10019": "Konnte L3-Netzwerk nicht vom VPC-Router [uuid:%s] trennen, da die Zustände von Master und Slave inkonsistent sind", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "Virtuelles Maschinengerät mit UUID: %s nicht gefunden", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "TPM-/host-file-Operation des VM-Geräts fehlgeschlagen: %s", + "ORG_ZSTACK_KVM_10163": "KVM-vTPM-/host-file-Operation fehlgeschlagen: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "Volume-Protokolloperation fehlgeschlagen: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "Volume-Snapshot-Operation fehlgeschlagen: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "Kann keinen Backup-Speicher finden, um das Image [%s] auf den Primärspeicher [%s] herunterzuladen, da kein bereitgestelltes und zugängliches Image vorhanden ist", "ORG_ZSTACK_VPC_10018": "Konnte L3-Netzwerk nicht vom VPC-Router [uuid:%s] trennen, da sein Zustand nicht aktiv oder gestoppt ist", "ORG_ZSTACK_ALIYUN_CORE_10037": "Das Image existiert bereits remote; bitte verwenden Sie zunächst sync, um die Konsistenz sicherzustellen.", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-en_US.json b/conf/i18n/globalErrorCodeMapping/global-error-en_US.json index 8fda69f4152..161b0bf5a3b 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-en_US.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-en_US.json @@ -488,6 +488,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "cannot find virtual machine with UUID: %s", "ORG_ZSTACK_VPC_10019": "could not detach L3 network from VPC router[uuid:%s] because the states of the master and slave are inconsistent", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "cannot find virtual machine device with uuid: %s", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "VM device TPM/host-file operation failed: %s", + "ORG_ZSTACK_KVM_10163": "KVM vTPM/host-file operation failed: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "volume protocol operation failed: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "volume snapshot operation failed: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "cannot find backup storage to download image [%s] to primary storage [%s] due to lack of ready and accessible image", "ORG_ZSTACK_VPC_10018": "could not detach L3 network from VPC router[uuid:%s] as its state is not running or stopped", "ORG_ZSTACK_ALIYUN_CORE_10037": "The image already existed remotely; please use sync first to ensure consistency.", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-fr-FR.json b/conf/i18n/globalErrorCodeMapping/global-error-fr-FR.json index a0e299aff9f..eb0bb3452b5 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-fr-FR.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-fr-FR.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "impossible de trouver la machine virtuelle avec l'UUID : %s", "ORG_ZSTACK_VPC_10019": "impossible de détacher le réseau L3 du routeur VPC [uuid:%s] car les états du maître et de l'esclave sont incohérents", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "impossible de trouver le dispositif de machine virtuelle avec l'uuid : %s", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "échec de l'opération TPM/host-file du périphérique de VM : %s", + "ORG_ZSTACK_KVM_10163": "échec de l'opération KVM vTPM/host-file : %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "échec de l'opération du protocole de volume : %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "échec de l'opération de snapshot de volume : %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "impossible de trouver un stockage de sauvegarde pour télécharger l'image [%s] vers le stockage principal [%s] en raison de l'absence d'image prête et accessible", "ORG_ZSTACK_VPC_10018": "impossible de détacher le réseau L3 du routeur VPC [uuid:%s] car son état n'est pas en cours d'exécution ou arrêté", "ORG_ZSTACK_ALIYUN_CORE_10037": "L'image existe déjà à distance ; veuillez d'abord utiliser sync pour assurer la cohérence.", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-id-ID.json b/conf/i18n/globalErrorCodeMapping/global-error-id-ID.json index d27b78d3499..12c632427e9 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-id-ID.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-id-ID.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "tidak dapat menemukan mesin virtual dengan UUID: %s", "ORG_ZSTACK_VPC_10019": "tidak dapat melepaskan jaringan L3 dari router VPC [uuid:%s] karena status master dan slave tidak konsisten", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "tidak dapat menemukan perangkat mesin virtual dengan uuid: %s", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "operasi TPM/host-file perangkat VM gagal: %s", + "ORG_ZSTACK_KVM_10163": "operasi KVM vTPM/host-file gagal: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "operasi protokol volume gagal: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "operasi snapshot volume gagal: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "tidak dapat menemukan penyimpanan cadangan untuk mengunduh gambar [%s] ke penyimpanan primer [%s] karena tidak ada gambar yang siap dan dapat diakses", "ORG_ZSTACK_VPC_10018": "tidak dapat melepaskan jaringan L3 dari router VPC [uuid:%s] karena statusnya tidak berjalan atau berhenti", "ORG_ZSTACK_ALIYUN_CORE_10037": "gambar sudah ada secara remote; harap gunakan sync terlebih dahulu untuk memastikan konsistensi", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-ja-JP.json b/conf/i18n/globalErrorCodeMapping/global-error-ja-JP.json index 4acbf1c06e4..8c456addbf3 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-ja-JP.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-ja-JP.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "UUID:%sの仮想マシンが見つかりません", "ORG_ZSTACK_VPC_10019": "マスターとスレーブの状態が一致しないため、VPCルーター[uuid:%s]からL3ネットワークをデタッチできませんでした", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "uuid:%sの仮想マシンデバイスが見つかりません", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "VMデバイスのTPM/host-file操作に失敗しました: %s", + "ORG_ZSTACK_KVM_10163": "KVM vTPM/host-file操作に失敗しました: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "ボリュームプロトコル操作に失敗しました: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "ボリュームスナップショット操作に失敗しました: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "準備ができてアクセス可能な画像がないため、画像を[%s]からプライマリストレージ[%s]にダウンロードするためのバックアップストレージが見つかりません", "ORG_ZSTACK_VPC_10018": "状態が実行中または停止していないため、VPCルーター[uuid:%s]からL3ネットワークをデタッチできませんでした", "ORG_ZSTACK_ALIYUN_CORE_10037": "画像はリモートに既に存在します。整合性を確保するために、まず同期を使用してください。", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-ko-KR.json b/conf/i18n/globalErrorCodeMapping/global-error-ko-KR.json index d9d5de55de9..a7da8dcf59e 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-ko-KR.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-ko-KR.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "UUID가 %s인 가상 머신을 찾을 수 없습니다", "ORG_ZSTACK_VPC_10019": "마스터와 슬레이브의 상태가 일치하지 않아 VPC 라우터[uuid:%s]에서 L3 네트워크를 분리할 수 없습니다", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "uuid가 %s인 가상 머신 디바이스를 찾을 수 없습니다", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "VM 디바이스 TPM/host-file 작업에 실패했습니다: %s", + "ORG_ZSTACK_KVM_10163": "KVM vTPM/host-file 작업에 실패했습니다: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "볼륨 프로토콜 작업에 실패했습니다: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "볼륨 스냅샷 작업에 실패했습니다: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "준비되고 접근 가능한 이미지가 부족하여 이미지 [%s]를 주 스토리지 [%s]로 다운로드할 백업 스토리지를 찾을 수 없습니다", "ORG_ZSTACK_VPC_10018": "VPC 라우터[uuid:%s]의 상태가 실행 중이거나 중지되어 있지 않아 L3 네트워크를 분리할 수 없습니다", "ORG_ZSTACK_ALIYUN_CORE_10037": "이미지가 원격지에 존재합니다. 동기화(sync)를 먼저 사용하여 일관성을 확보하세요.", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-ru-RU.json b/conf/i18n/globalErrorCodeMapping/global-error-ru-RU.json index 2bcbcfe039e..63d3868899b 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-ru-RU.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-ru-RU.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "не удается найти виртуальную машину с UUID: %s", "ORG_ZSTACK_VPC_10019": "не удалось отсоединить сеть L3 от VPC-роутера [uuid:%s], так как состояния основного и резервного узлов несовместимы", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "не удается найти устройство виртуальной машины с uuid: %s", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "операция TPM/host-file устройства ВМ не удалась: %s", + "ORG_ZSTACK_KVM_10163": "операция KVM vTPM/host-file не удалась: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "операция протокола тома не удалась: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "операция снимка тома не удалась: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "не удается найти резервное хранилище для загрузки образа [%s] в первичное хранилище [%s] из-за отсутствия готового и доступного образа", "ORG_ZSTACK_VPC_10018": "не удалось отсоединить сеть L3 от VPC-роутера [uuid:%s], так как его состояние не является запущенным или остановленным", "ORG_ZSTACK_ALIYUN_CORE_10037": "образ уже существует удаленно; сначала используйте sync для обеспечения согласованности", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-th-TH.json b/conf/i18n/globalErrorCodeMapping/global-error-th-TH.json index 3c8c04b2cf7..8f88ad642a5 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-th-TH.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-th-TH.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "", "ORG_ZSTACK_VPC_10019": "ไม่พบเครื่องเสมือนที่มี UUID: %s", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "การดำเนินการ TPM/host-file ของอุปกรณ์ VM ล้มเหลว: %s", + "ORG_ZSTACK_KVM_10163": "การดำเนินการ KVM vTPM/host-file ล้มเหลว: %s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "การดำเนินการโปรโตคอลของโวลุ่มล้มเหลว: %s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "การดำเนินการสแนปช็อตของโวลุ่มล้มเหลว: %s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "ไม่สามารถถอดเครือขาย L3 จากเราเตอร์ VPC [uuid:%s] ได้เนื่องจากสถานะของ master และ slave ไม่สอดคล้องกัน", "ORG_ZSTACK_VPC_10018": "", "ORG_ZSTACK_ALIYUN_CORE_10037": "ไม่พบอุปกรณ์เครื่องเสมือนที่มี uuid: %s", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json b/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json index 0c36ae4980b..26a7beb03b4 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-zh_CN.json @@ -488,6 +488,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "无法找到具有 UUID：%s 的 VM", "ORG_ZSTACK_VPC_10019": "无法将三层网络从uuid:%s的VPC路由器解绑，因为主从状态不一致", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "无法找到具有 UUID：%s 的虚拟机设备", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "虚拟机设备 TPM/host-file 操作失败：%s", + "ORG_ZSTACK_KVM_10163": "KVM vTPM/host-file 操作失败：%s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "云盘协议操作失败：%s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "云盘快照操作失败：%s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "无法找到备份存储以将图像 [%s] 下载到主存储 [%s]，原因是没有可用的 Ready 和可访问的图像", "ORG_ZSTACK_VPC_10018": "无法将三层网络从VPC路由器[uuid:%s]解绑，因为其状态不是运行中或已停止", "ORG_ZSTACK_ALIYUN_CORE_10037": "镜像已在远程存在，请先使用同步。", diff --git a/conf/i18n/globalErrorCodeMapping/global-error-zh_TW.json b/conf/i18n/globalErrorCodeMapping/global-error-zh_TW.json index 16f440db0ea..c304c781941 100644 --- a/conf/i18n/globalErrorCodeMapping/global-error-zh_TW.json +++ b/conf/i18n/globalErrorCodeMapping/global-error-zh_TW.json @@ -484,6 +484,10 @@ "ORG_ZSTACK_COMPUTE_VM_DEVICES_10007": "無法找到具有 UUID：%s 的 VM", "ORG_ZSTACK_VPC_10019": "無法将三层网络从uuid:%s的VPC路由器解綁，因为主从状态不一致", "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008": "無法找到具有 UUID：%s 的虚拟機設备", + "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009": "虛擬機設備 TPM/host-file 操作失敗：%s", + "ORG_ZSTACK_KVM_10163": "KVM vTPM/host-file 操作失敗：%s", + "ORG_ZSTACK_STORAGE_VOLUME_10101": "雲盤協議操作失敗：%s", + "ORG_ZSTACK_STORAGE_SNAPSHOT_10030": "雲盤快照操作失敗：%s", "ORG_ZSTACK_STORAGE_CEPH_PRIMARY_10030": "無法找到备份儲儲以将圖像 [%s] 下载到主儲儲 [%s]，原因是没有可用的 Ready 和可访问的圖像", "ORG_ZSTACK_VPC_10018": "無法将三层网络从VPC路由器[uuid:%s]解綁，因为其状态不是運行中或已停止", "ORG_ZSTACK_ALIYUN_CORE_10037": "镜像已在远程儲在，請先使用同步。", diff --git a/conf/persistence.xml b/conf/persistence.xml index e76141af3ad..f24361abbfa 100755 --- a/conf/persistence.xml +++ b/conf/persistence.xml @@ -18,6 +18,11 @@ org.zstack.resourceconfig.ResourceConfigVO org.zstack.header.managementnode.ManagementNodeVO org.zstack.header.managementnode.ManagementNodeContextVO + org.zstack.header.tpm.entity.TpmVO + org.zstack.header.tpm.entity.TpmKeyBackupVO + org.zstack.header.vm.additions.VmHostFileVO + org.zstack.header.vm.additions.VmHostBackupFileVO + org.zstack.header.vm.additions.VmHostFileContentVO org.zstack.header.zone.ZoneVO org.zstack.header.zone.ZoneEO org.zstack.header.cluster.ClusterVO @@ -25,6 +30,7 @@ org.zstack.header.host.HostVO org.zstack.header.host.HostEO org.zstack.header.host.HostNetworkLabelVO + org.zstack.header.host.HostKeyIdentityVO org.zstack.header.host.CpuFeaturesHistoryVO org.zstack.header.storage.primary.PrimaryStorageVO org.zstack.header.storage.primary.PrimaryStorageEO @@ -191,6 +197,11 @@ org.zstack.header.vm.VmInstanceNumaNodeVO org.zstack.header.host.HostNumaNodeVO org.zstack.header.core.encrypt.EncryptionIntegrityVO + org.zstack.header.keyprovider.KeyProviderVO + org.zstack.header.keyprovider.KmsVO + org.zstack.header.keyprovider.KmsIdentityVO + org.zstack.header.keyprovider.NkpVO + org.zstack.header.keyprovider.EncryptedResourceKeyRefVO org.zstack.storage.primary.sharedblock.SharedBlockCapacityVO org.zstack.header.vm.devices.VmInstanceDeviceAddressVO org.zstack.header.vm.devices.VmInstanceDeviceAddressArchiveVO diff --git a/conf/serviceConfig/tpm.xml b/conf/serviceConfig/tpm.xml new file mode 100644 index 00000000000..dab0467a8f6 --- /dev/null +++ b/conf/serviceConfig/tpm.xml @@ -0,0 +1,22 @@ + + + tpm + TpmApiInterceptor + + + org.zstack.header.tpm.api.APIAddTpmMsg + + + org.zstack.header.tpm.api.APIGetTpmCapabilityMsg + + + org.zstack.header.tpm.api.APIQueryTpmMsg + query + + + org.zstack.header.tpm.api.APIRemoveTpmMsg + + + org.zstack.header.tpm.api.APIUpdateTpmMsg + + diff --git a/conf/springConfigXml/Kvm.xml b/conf/springConfigXml/Kvm.xml index a96965fc01b..8ac96d7f14a 100755 --- a/conf/springConfigXml/Kvm.xml +++ b/conf/springConfigXml/Kvm.xml @@ -134,6 +134,12 @@ + + + + + + @@ -171,9 +177,10 @@ - + + @@ -261,4 +268,57 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/springConfigXml/VmInstanceManager.xml b/conf/springConfigXml/VmInstanceManager.xml index 09a9cb8d0e5..fe980440eb8 100755 --- a/conf/springConfigXml/VmInstanceManager.xml +++ b/conf/springConfigXml/VmInstanceManager.xml @@ -118,11 +118,12 @@ org.zstack.compute.vm.VmExpungeSdnNicFlow - org.zstack.compute.vm.VmExpungeRootVolumeFlow - org.zstack.compute.vm.VmExpungeMemoryVolumeFlow - org.zstack.compute.vm.VmExpungeCacheVolumeFlow - - + org.zstack.compute.vm.VmExpungeRootVolumeFlow + org.zstack.compute.vm.VmExpungeMemoryVolumeFlow + org.zstack.compute.vm.VmExpungeCacheVolumeFlow + org.zstack.compute.vm.VmExpungeVmResourceCascadeFlow + + org.zstack.compute.vm.PauseVmOnHypervisorFlow @@ -273,4 +274,56 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/main/java/org/zstack/core/cascade/CascadeConstant.java b/core/src/main/java/org/zstack/core/cascade/CascadeConstant.java index 59557ebd199..f3ca6096561 100755 --- a/core/src/main/java/org/zstack/core/cascade/CascadeConstant.java +++ b/core/src/main/java/org/zstack/core/cascade/CascadeConstant.java @@ -11,5 +11,11 @@ public interface CascadeConstant { String DELETION_FORCE_DELETE_CODE = "deletion.forceDelete"; String DELETION_CLEANUP_CODE = "deletion.cleanup"; + /** + * Final removal of VM-associated resources (TPM, NvRam host files, etc.) when the VM row is expunged. + * Invoked from {@code APIExpungeVmInstanceMsg}; not part of {@link #DELETION_CODES}. + */ + String VM_INSTANCE_EXPUNGE_CODE = "vm.instance.expunge"; + List DELETION_CODES = Arrays.asList(DELETION_CHECK_CODE, DELETION_DELETE_CODE, DELETION_FORCE_DELETE_CODE); } diff --git a/core/src/main/java/org/zstack/core/log/LogSafeGson.java b/core/src/main/java/org/zstack/core/log/LogSafeGson.java index 13456c13bc0..7031e656599 100644 --- a/core/src/main/java/org/zstack/core/log/LogSafeGson.java +++ b/core/src/main/java/org/zstack/core/log/LogSafeGson.java @@ -99,6 +99,10 @@ String getMaskedValue(String raw) { return "*****"; } else if (annotation.type().tag()) { return tagInfoHider.call(raw); + } else if (annotation.type().longText()) { + if (raw == null) + return null; + return (raw.length() < 20) ? raw : raw.substring(0, 10) + "..." + raw.substring(raw.length() - 10); } else { return Utils.getLogMaskWords().getOrDefault(raw, uriPattern.matcher(raw).replaceFirst(":*****@")); } diff --git a/core/src/main/java/org/zstack/core/workflow/SimpleFlowChain.java b/core/src/main/java/org/zstack/core/workflow/SimpleFlowChain.java index a78a2172e97..4294c600ca7 100755 --- a/core/src/main/java/org/zstack/core/workflow/SimpleFlowChain.java +++ b/core/src/main/java/org/zstack/core/workflow/SimpleFlowChain.java @@ -15,6 +15,7 @@ import org.zstack.core.errorcode.ErrorFacade; import org.zstack.core.telemetry.TelemetryFacade; import org.zstack.core.telemetry.TelemetryGlobalProperty; +import org.zstack.header.core.AsyncBackup; import org.zstack.header.core.workflow.*; import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.errorcode.OperationFailureException; @@ -31,6 +32,7 @@ import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; import static org.zstack.core.Platform.inerr; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; @@ -71,6 +73,7 @@ public class SimpleFlowChain implements FlowTrigger, FlowRollback, FlowChain, Fl private List> afterDone = new ArrayList<>(); private List> afterError = new ArrayList<>(); private List> afterFinal = new ArrayList<>(); + private List asyncBackups = new ArrayList<>(); private boolean isFailCalled; @@ -291,11 +294,20 @@ public SimpleFlowChain() { id = "FCID_" + Platform.getUuid().substring(0, 8); } + public SimpleFlowChain(String name) { + this(); + this.name = name; + } + public SimpleFlowChain(Map data) { id = "FCID_" + Platform.getUuid().substring(0, 8); this.data.putAll(data); } + public static SimpleFlowChain of(String chainName) { + return new SimpleFlowChain(chainName); + } + @Override public List getFlows() { return flows; @@ -379,6 +391,10 @@ public SimpleFlowChain then(Flow flow) { return this; } + public SimpleFlowChain then(String flowName, Consumer consumer) { + return then(Flow.of(flowName).handle(consumer).build()); + } + public SimpleFlowChain ctxHandler(FlowContextHandler handler) { DebugUtils.Assert(contextHandler == null, "there has been an FlowContextHandler installed"); contextHandler = handler; @@ -391,6 +407,23 @@ public SimpleFlowChain error(FlowErrorHandler handler) { return this; } + @SuppressWarnings("rawtypes") + public SimpleFlowChain error(Consumer handler) { + DebugUtils.Assert(!asyncBackups.isEmpty(), + "propagateExceptionTo() must be called before error(Consumer)." + + " Call .propagateExceptionTo(completion) or .propagateExceptionTo((AsyncBackup) null) if you don't need async backup"); + AsyncBackup firstAsyncBackup = this.asyncBackups.get(0); + AsyncBackup[] otherAsyncBackups = this.asyncBackups.subList(1, this.asyncBackups.size()).toArray(new AsyncBackup[0]); + + DebugUtils.Assert(handler != null, "handler of errorHandler should not be null"); + return error(new FlowErrorHandler(firstAsyncBackup, otherAsyncBackups) { + @Override + public void handle(ErrorCode errCode, Map data) { + handler.accept(errCode); + } + }); + } + @Override public FlowChain Finally(FlowFinallyHandler handler) { finallyHandler = handler; @@ -433,6 +466,21 @@ public Map getData() { return this.data; } + public SimpleFlowChain propagateExceptionTo(AsyncBackup... backups) { + DebugUtils.Assert(backups != null, "backups in methods propagateExceptionTo() must be not null"); + DebugUtils.Assert(doneHandler == null, "propagateExceptionTo() must be called before SimpleFlowChain.done()"); + DebugUtils.Assert(errorHandler == null, "propagateExceptionTo() must be called before SimpleFlowChain.error()"); + DebugUtils.Assert(finallyHandler == null, "propagateExceptionTo() must be called before SimpleFlowChain.Finally()"); + + this.asyncBackups.addAll(Arrays.asList(backups)); + this.asyncBackups.removeIf(Objects::isNull); + + if (this.asyncBackups.isEmpty()) { + this.asyncBackups.add(null); + } + return this; + } + @Override public SimpleFlowChain done(FlowDoneHandler handler) { DebugUtils.Assert(doneHandler == null, "there has been a FlowDoneHandler installed"); @@ -440,6 +488,23 @@ public SimpleFlowChain done(FlowDoneHandler handler) { return this; } + @SuppressWarnings("rawtypes") + public SimpleFlowChain done(Runnable runnable) { + DebugUtils.Assert(!asyncBackups.isEmpty(), + "propagateExceptionTo() must be called before done(Runnable)." + + " Call .propagateExceptionTo(completion) or .propagateExceptionTo((AsyncBackup) null) if you don't need async backup"); + AsyncBackup firstAsyncBackup = this.asyncBackups.get(0); + AsyncBackup[] otherAsyncBackups = this.asyncBackups.subList(1, this.asyncBackups.size()).toArray(new AsyncBackup[0]); + + DebugUtils.Assert(runnable != null, "runnable of doneHandler should not be null"); + return done(new FlowDoneHandler(firstAsyncBackup, otherAsyncBackups) { + @Override + public void handle(Map data) { + runnable.run(); + } + }); + } + private void collectAfterRunnable(Flow flow) { List ad = FieldUtils.getAnnotatedFieldsOnThisClass(AfterDone.class, flow.getClass()); for (Field f : ad) { diff --git a/header/src/main/java/org/zstack/header/core/workflow/Flow.java b/header/src/main/java/org/zstack/header/core/workflow/Flow.java index 7fa262ded82..33f8d2ad8f4 100755 --- a/header/src/main/java/org/zstack/header/core/workflow/Flow.java +++ b/header/src/main/java/org/zstack/header/core/workflow/Flow.java @@ -1,6 +1,12 @@ package org.zstack.header.core.workflow; +import org.zstack.utils.DebugUtils; +import org.zstack.utils.FieldUtils; + import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Predicate; public interface Flow { void run(FlowTrigger trigger, Map data); @@ -10,4 +16,103 @@ public interface Flow { default boolean skip(Map data) { return false; } + + default String name() { + String innerName = FieldUtils.getFieldValue("__name__", this); + if (innerName != null && !innerName.trim().isEmpty()) { + return innerName; + } + return String.format("%s", this.getClass().getSimpleName()); + } + + @SuppressWarnings("rawtypes") + class FlowBuilder { + private final String flowName; + private Predicate skipPredicate; + private BiConsumer triggerConsumer; + private BiConsumer rollbackConsumer; + + private FlowBuilder(String flowName) { + DebugUtils.Assert(flowName != null, "flowName should not be null"); + this.flowName = flowName; + } + + public FlowBuilder skipIf(Predicate predicate) { + DebugUtils.Assert(predicate != null, "skipPredicate should not be null"); + this.skipPredicate = predicate; + return this; + } + + public FlowBuilder runIf(Predicate predicate) { + DebugUtils.Assert(predicate != null, "runIf predicate should not be null"); + this.skipPredicate = predicate.negate(); + return this; + } + + public FlowBuilder handle(BiConsumer consumer) { + DebugUtils.Assert(consumer != null, "handle consumer should not be null"); + this.triggerConsumer = consumer; + return this; + } + + public FlowBuilder handle(Consumer consumer) { + DebugUtils.Assert(consumer != null, "handle consumer should not be null"); + this.triggerConsumer = (trigger, data) -> consumer.accept(trigger); + return this; + } + + public FlowBuilder rollback(BiConsumer consumer) { + DebugUtils.Assert(consumer != null, "rollback consumer should not be null"); + this.rollbackConsumer = consumer; + return this; + } + + public FlowBuilder rollback(Consumer consumer) { + DebugUtils.Assert(consumer != null, "rollback consumer should not be null"); + this.rollbackConsumer = (trigger, data) -> consumer.accept(trigger); + return this; + } + + public Flow build() { + DebugUtils.Assert(triggerConsumer != null, "handle() must be called before build()"); + Predicate finalSkipPredicate = skipPredicate; + BiConsumer finalTriggerConsumer = triggerConsumer; + BiConsumer finalRollbackConsumer = rollbackConsumer; + + return new Flow() { + @Override + public void run(FlowTrigger trigger, Map data) { + finalTriggerConsumer.accept(trigger, data); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + if (finalRollbackConsumer == null) { + trigger.rollback(); + } else { + finalRollbackConsumer.accept(trigger, data); + } + } + + @Override + public boolean skip(Map data) { + return finalSkipPredicate != null && finalSkipPredicate.test(data); + } + + @Override + public String name() { + return flowName; + } + + @Override + public String toString() { + return name(); + } + }; + } + } + + static FlowBuilder of(String flowName) { + return new FlowBuilder(flowName); + } } diff --git a/header/src/main/java/org/zstack/header/errorcode/SysErrors.java b/header/src/main/java/org/zstack/header/errorcode/SysErrors.java index 634f08e17fe..47c85dd72dd 100755 --- a/header/src/main/java/org/zstack/header/errorcode/SysErrors.java +++ b/header/src/main/java/org/zstack/header/errorcode/SysErrors.java @@ -20,7 +20,9 @@ public enum SysErrors { UNIMPLEMENTED_OPERATION_ERROR(1014), HTTP_ERROR(1015), IO_ERROR(1016), - CANCEL_ERROR(1017); + CANCEL_ERROR(1017), + NOT_SUPPORTED(1091), + ; private String code; diff --git a/header/src/main/java/org/zstack/header/host/HostKeyIdentityVO.java b/header/src/main/java/org/zstack/header/host/HostKeyIdentityVO.java new file mode 100644 index 00000000000..1433e075954 --- /dev/null +++ b/header/src/main/java/org/zstack/header/host/HostKeyIdentityVO.java @@ -0,0 +1,89 @@ +package org.zstack.header.host; + +import org.zstack.header.vo.EntityGraph; +import org.zstack.header.vo.ForeignKey; + +import javax.persistence.*; +import java.sql.Timestamp; + +@Entity +@Table +@EntityGraph( + parents = { + @EntityGraph.Neighbour(type = HostVO.class, myField = "hostUuid", targetField = "uuid") + } +) +public class HostKeyIdentityVO { + @Id + @Column + @ForeignKey(parentEntityClass = HostEO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String hostUuid; + + @Column + private String publicKey; + + @Column + private String fingerprint; + + @Column + private Boolean verified = false; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @PreUpdate + private void preUpdate() { + lastOpDate = null; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getPublicKey() { + return publicKey; + } + + public void setPublicKey(String publicKey) { + this.publicKey = publicKey; + } + + public String getFingerprint() { + return fingerprint; + } + + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + public Boolean getVerified() { + return verified; + } + + public void setVerified(Boolean verified) { + this.verified = verified; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/header/src/main/java/org/zstack/header/host/HostKeyIdentityVO_.java b/header/src/main/java/org/zstack/header/host/HostKeyIdentityVO_.java new file mode 100644 index 00000000000..a2bcfa7b850 --- /dev/null +++ b/header/src/main/java/org/zstack/header/host/HostKeyIdentityVO_.java @@ -0,0 +1,15 @@ +package org.zstack.header.host; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(HostKeyIdentityVO.class) +public class HostKeyIdentityVO_ { + public static volatile SingularAttribute hostUuid; + public static volatile SingularAttribute publicKey; + public static volatile SingularAttribute fingerprint; + public static volatile SingularAttribute verified; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/keyprovider/EncryptedResourceKeyManager.java b/header/src/main/java/org/zstack/header/keyprovider/EncryptedResourceKeyManager.java new file mode 100644 index 00000000000..ffa38675402 --- /dev/null +++ b/header/src/main/java/org/zstack/header/keyprovider/EncryptedResourceKeyManager.java @@ -0,0 +1,182 @@ +package org.zstack.header.keyprovider; + +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.core.Completion; + +/** + * Unified resource key management service. + * Business layers (TPM, volume encryption, etc.) call this to create/retrieve DEKs + * without knowing gRPC/key-tools protocol details. + */ +public interface EncryptedResourceKeyManager { + + /** + * Get or create a resource encryption key. + *

+ * Semantically reuses the existing key record for the same {@code (resourceType, resourceUuid)} + * when one is already available; otherwise creates a new one and returns the plaintext DEK. + *

+ * This contract does not guarantee concurrent linearizability by itself. Callers must not assume + * the interface alone provides serialization, uniqueness enforcement, or transaction-level protection + * across concurrent create requests. + * + * @param ctx context describing the resource and key provider + * @param completion returns {@link ResourceKeyResult} containing the plaintext DEK (base64) + */ + void getOrCreateKey(GetOrCreateResourceKeyContext ctx, + ReturnValueCompletion completion); + + /** + * Load the existing resource encryption key material only. + *

+ * Requires an {@code EncryptedResourceKeyRef} row and a usable secret reference already stored + * for the resource. Does not insert a ref row and does not call + * key-tool/KMS create APIs. + *

+ * The implementation may still call key-tool/KMS get/unwrap for the existing + * secret ref in order to return the plaintext DEK (for example defining the secret on the destination + * host during hot migration). That RPC is read-side materialization, not secret creation. + *

+ * On success, the implementation may update {@code EncryptedResourceKeyRef} provider columns to match + * the resolved key provider when they have drifted (same behavior as the existing-key branch of + * {@link #getOrCreateKey}). + * + * @param ctx same fields as {@link #getOrCreateKey}; identifies resource and provider + * @return {@link ResourceKeyResult} with {@code createdNewKey == false} on success + * @throws org.zstack.header.errorcode.OperationFailureException when the key cannot be loaded + */ + ResourceKeyResult getKey(GetOrCreateResourceKeyContext ctx); + + /** + * Roll back a newly created resource key during upper-layer workflow rollback. + *

+ * When {@link ResourceKeyResult#isCreatedNewKey()} is true, the implementation deletes the + * key-tool secret if one was materialized, then removes the {@code EncryptedResourceKeyRef} row + * for the resource (same storage effect as detaching the key provider from the resource). + * When {@code createdNewKey} is false (existing secret was reused), this is a no-op. + */ + void rollbackCreatedKey(ResourceKeyResult result, Completion completion); + + class GetOrCreateResourceKeyContext { + private String resourceUuid; + private String resourceType; + private String keyProviderUuid; + private String keyProviderName; + private String purpose; + + public String getResourceUuid() { + return resourceUuid; + } + + public void setResourceUuid(String resourceUuid) { + this.resourceUuid = resourceUuid; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + public String getKeyProviderName() { + return keyProviderName; + } + + public void setKeyProviderName(String keyProviderName) { + this.keyProviderName = keyProviderName; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + } + + class ResourceKeyResult { + private String resourceUuid; + private String resourceType; + private String keyProviderUuid; + private String keyProviderName; + private Integer keyVersion; + private String dekBase64; + private String secretRef; + private boolean createdNewKey; + + public String getResourceUuid() { + return resourceUuid; + } + + public void setResourceUuid(String resourceUuid) { + this.resourceUuid = resourceUuid; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + public String getKeyProviderName() { + return keyProviderName; + } + + public void setKeyProviderName(String keyProviderName) { + this.keyProviderName = keyProviderName; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getDekBase64() { + return dekBase64; + } + + public void setDekBase64(String dekBase64) { + this.dekBase64 = dekBase64; + } + + public String getSecretRef() { + return secretRef; + } + + public void setSecretRef(String secretRef) { + this.secretRef = secretRef; + } + + public boolean isCreatedNewKey() { + return createdNewKey; + } + + public void setCreatedNewKey(boolean createdNewKey) { + this.createdNewKey = createdNewKey; + } + } +} diff --git a/header/src/main/java/org/zstack/header/keyprovider/KeyProviderRekeyAssociationExtensionPoint.java b/header/src/main/java/org/zstack/header/keyprovider/KeyProviderRekeyAssociationExtensionPoint.java new file mode 100644 index 00000000000..e23f704b259 --- /dev/null +++ b/header/src/main/java/org/zstack/header/keyprovider/KeyProviderRekeyAssociationExtensionPoint.java @@ -0,0 +1,11 @@ +package org.zstack.header.keyprovider; + +import java.util.List; + +public interface KeyProviderRekeyAssociationExtensionPoint { + String getType(); + + String getAssociatedResourceType(); + + List getAssociatedResourceUuids(List resourceUuids); +} diff --git a/header/src/main/java/org/zstack/header/log/NoLogging.java b/header/src/main/java/org/zstack/header/log/NoLogging.java index b53c90f6333..5454acc849d 100644 --- a/header/src/main/java/org/zstack/header/log/NoLogging.java +++ b/header/src/main/java/org/zstack/header/log/NoLogging.java @@ -26,7 +26,9 @@ public boolean mask() { enum Type { Simple, Tag, - Uri; + Uri, + LongText, + ; public boolean simple() { return this == Simple; @@ -39,6 +41,10 @@ public boolean tag() { public boolean uri() { return this == Uri; } + + public boolean longText() { + return this == LongText; + } } Behavior behavior() default Behavior.Mask; diff --git a/header/src/main/java/org/zstack/header/message/DocUtils.java b/header/src/main/java/org/zstack/header/message/DocUtils.java index 5e9957cf4d8..39c25faf252 100644 --- a/header/src/main/java/org/zstack/header/message/DocUtils.java +++ b/header/src/main/java/org/zstack/header/message/DocUtils.java @@ -1,5 +1,8 @@ package org.zstack.header.message; +import org.zstack.utils.StringDSL; + +import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -30,4 +33,12 @@ public static String uuidForAPIDoc(){ public static void removeApiUuidMap(String apiName){ apiUuidMap.remove(apiName); } + + public static String createFixedUuid(Class voClass) { + return StringDSL.createFixedUuid(voClass); + } + + public static Timestamp timestamp() { + return new Timestamp(date); + } } diff --git a/header/src/main/java/org/zstack/header/message/Message.java b/header/src/main/java/org/zstack/header/message/Message.java index 42f9fcc6f08..1b2db87baaf 100755 --- a/header/src/main/java/org/zstack/header/message/Message.java +++ b/header/src/main/java/org/zstack/header/message/Message.java @@ -54,6 +54,10 @@ protected static String uuid() { } } + protected static String uuid(Class voClass) { + return DocUtils.createFixedUuid(voClass); + } + public Message() { createdTime = System.currentTimeMillis(); id = UUID.randomUUID().toString().replace("-", ""); diff --git a/header/src/main/java/org/zstack/header/secret/ResolveVtpmLibvirtSecretOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/secret/ResolveVtpmLibvirtSecretOnHypervisorMsg.java new file mode 100644 index 00000000000..e4ec14cb72f --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/ResolveVtpmLibvirtSecretOnHypervisorMsg.java @@ -0,0 +1,26 @@ +package org.zstack.header.secret; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.message.NeedReplyMessage; + +public class ResolveVtpmLibvirtSecretOnHypervisorMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + private String vmUuid; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/ResolveVtpmLibvirtSecretOnHypervisorReply.java b/header/src/main/java/org/zstack/header/secret/ResolveVtpmLibvirtSecretOnHypervisorReply.java new file mode 100644 index 00000000000..af9927592d5 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/ResolveVtpmLibvirtSecretOnHypervisorReply.java @@ -0,0 +1,15 @@ +package org.zstack.header.secret; + +import org.zstack.header.message.MessageReply; + +public class ResolveVtpmLibvirtSecretOnHypervisorReply extends MessageReply { + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostDefineMsg.java b/header/src/main/java/org/zstack/header/secret/SecretHostDefineMsg.java new file mode 100644 index 00000000000..d3258f249fd --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostDefineMsg.java @@ -0,0 +1,87 @@ +package org.zstack.header.secret; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.log.NoLogging; +import org.zstack.header.message.NeedReplyMessage; + +/** + * Request to ensure secret on KVM host (for VM e.g. vTPM). + * Caller provides plaintext DEK (dekBase64), then host seals it with host public key + * and forwards the envelope to key-agent. + */ +public class SecretHostDefineMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + @NoLogging + private String dekBase64; + private String vmUuid; + private String purpose; + private Integer keyVersion; + private String usageInstance; + private String secretUuid; + private String description; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getDekBase64() { + return dekBase64; + } + + public void setDekBase64(String dekBase64) { + this.dekBase64 = dekBase64; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getUsageInstance() { + return usageInstance; + } + + public void setUsageInstance(String usageInstance) { + this.usageInstance = usageInstance; + } + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostDefineReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostDefineReply.java new file mode 100644 index 00000000000..13ef13c07df --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostDefineReply.java @@ -0,0 +1,19 @@ +package org.zstack.header.secret; + +import org.zstack.header.message.MessageReply; + +/** Reply for SecretHostDefineMsg (define secret on host for VM e.g. vTPM). */ +public class SecretHostDefineReply extends MessageReply { + public static final String ERROR_CODE_KEYS_NOT_ON_DISK = "KEY_AGENT_KEYS_NOT_ON_DISK"; + public static final String ERROR_CODE_KEY_FILES_INTEGRITY_MISMATCH = "KEY_AGENT_KEY_FILES_INTEGRITY_MISMATCH"; + + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostDeleteMsg.java b/header/src/main/java/org/zstack/header/secret/SecretHostDeleteMsg.java new file mode 100644 index 00000000000..f9502d16c67 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostDeleteMsg.java @@ -0,0 +1,53 @@ +package org.zstack.header.secret; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.message.NeedReplyMessage; + +public class SecretHostDeleteMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + private String vmUuid; + private String purpose; + private Integer keyVersion; + private String usageInstance; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getUsageInstance() { + return usageInstance; + } + + public void setUsageInstance(String usageInstance) { + this.usageInstance = usageInstance; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostDeleteReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostDeleteReply.java new file mode 100644 index 00000000000..6640b5b60ca --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostDeleteReply.java @@ -0,0 +1,7 @@ +package org.zstack.header.secret; + +import org.zstack.header.message.MessageReply; + +/** Reply for SecretHostDeleteMsg. */ +public class SecretHostDeleteReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostGetMsg.java b/header/src/main/java/org/zstack/header/secret/SecretHostGetMsg.java new file mode 100644 index 00000000000..f63284880e0 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostGetMsg.java @@ -0,0 +1,56 @@ +package org.zstack.header.secret; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.message.NeedReplyMessage; + +/** + * Request to get an existing secret on KVM host by vmUuid, purpose and keyVersion. + */ +public class SecretHostGetMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + private String vmUuid; + private String purpose; + private Integer keyVersion; + private String usageInstance; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getUsageInstance() { + return usageInstance; + } + + public void setUsageInstance(String usageInstance) { + this.usageInstance = usageInstance; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java new file mode 100644 index 00000000000..cd474433110 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java @@ -0,0 +1,18 @@ +package org.zstack.header.secret; + +import org.zstack.header.message.MessageReply; + +/** Reply for SecretHostGetMsg. */ +public class SecretHostGetReply extends MessageReply { + public static final String ERROR_CODE_SECRET_NOT_FOUND = "KEY_AGENT_SECRET_NOT_FOUND"; + + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotMsg.java b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotMsg.java index 886f697c6df..919e0b9af8e 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotMsg.java @@ -16,6 +16,8 @@ public class CreateVolumesSnapshotMsg extends NeedReplyMessage implements NeedQu private List volumeSnapshotJobs; + private boolean backupHostFileIfNeeded; + public String getAccountUuid() { return accountUuid; } @@ -44,4 +46,12 @@ public ConsistentType getConsistentType() { public void setConsistentType(ConsistentType consistentType) { this.consistentType = consistentType; } + + public boolean isBackupHostFileIfNeeded() { + return backupHostFileIfNeeded; + } + + public void setBackupHostFileIfNeeded(boolean backupHostFileIfNeeded) { + this.backupHostFileIfNeeded = backupHostFileIfNeeded; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerMsg.java b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerMsg.java index 9e52d36b0d3..a9b4f6bdc62 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerMsg.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerMsg.java @@ -19,6 +19,8 @@ public class CreateVolumesSnapshotOverlayInnerMsg extends NeedReplyMessage imple private List lockedVolumeUuids; + private boolean backupHostFileIfNeeded; + public List getLockedVmInstanceUuids() { return lockedVmInstanceUuids; } @@ -63,4 +65,12 @@ public ConsistentType getConsistentType() { public void setConsistentType(ConsistentType consistentType) { this.consistentType = consistentType; } + + public boolean isBackupHostFileIfNeeded() { + return backupHostFileIfNeeded; + } + + public void setBackupHostFileIfNeeded(boolean backupHostFileIfNeeded) { + this.backupHostFileIfNeeded = backupHostFileIfNeeded; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerReply.java b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerReply.java index a488079afc2..cd816f6014c 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerReply.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotOverlayInnerReply.java @@ -9,6 +9,7 @@ */ public class CreateVolumesSnapshotOverlayInnerReply extends MessageReply { private List inventories; + private List hostBackupFileUuidList; public List getInventories() { return inventories; @@ -17,4 +18,12 @@ public List getInventories() { public void setInventories(List inventories) { this.inventories = inventories; } + + public List getHostBackupFileUuidList() { + return hostBackupFileUuidList; + } + + public void setHostBackupFileUuidList(List hostBackupFileUuidList) { + this.hostBackupFileUuidList = hostBackupFileUuidList; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotReply.java b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotReply.java index 41e1b0270f9..ac510f260b7 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotReply.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/CreateVolumesSnapshotReply.java @@ -9,6 +9,7 @@ */ public class CreateVolumesSnapshotReply extends MessageReply { private List inventories; + private List hostBackupFileUuidList; public List getInventories() { return inventories; @@ -17,4 +18,12 @@ public List getInventories() { public void setInventories(List inventories) { this.inventories = inventories; } + + public List getHostBackupFileUuidList() { + return hostBackupFileUuidList; + } + + public void setHostBackupFileUuidList(List hostBackupFileUuidList) { + this.hostBackupFileUuidList = hostBackupFileUuidList; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmMsg.java b/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmMsg.java index a54e8a2b8cf..c5199404ad7 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmMsg.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmMsg.java @@ -2,6 +2,7 @@ import org.zstack.header.host.HostMessage; import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.vm.additions.VmHostFileBackupJob; import java.util.List; @@ -11,6 +12,7 @@ public class TakeVolumesSnapshotOnKvmMsg extends NeedReplyMessage implements HostMessage { private List snapshotJobs; private String hostUuid; + private List vmHostFileBackupJobs; @Override public String getHostUuid() { @@ -28,4 +30,13 @@ public List getSnapshotJobs() { public void setSnapshotJobs(List snapshotJobs) { this.snapshotJobs = snapshotJobs; } + + public List getVmHostFileBackupJobs() { + return vmHostFileBackupJobs; + } + + public void setVmHostFileBackupJobs(List vmHostFileBackupJobs) { + this.vmHostFileBackupJobs = vmHostFileBackupJobs; + } + } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmReply.java b/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmReply.java index 2ba73944a57..bb63fc9d324 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmReply.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/TakeVolumesSnapshotOnKvmReply.java @@ -9,6 +9,7 @@ */ public class TakeVolumesSnapshotOnKvmReply extends MessageReply { private List snapshotsResults; + private String hostBackupTempResourceUuid; public List getSnapshotsResults() { return snapshotsResults; @@ -17,4 +18,12 @@ public List getSnapshotsResults() { public void setSnapshotsResults(List snapshotsResults) { this.snapshotsResults = snapshotsResults; } + + public String getHostBackupTempResourceUuid() { + return hostBackupTempResourceUuid; + } + + public void setHostBackupTempResourceUuid(String hostBackupTempResourceUuid) { + this.hostBackupTempResourceUuid = hostBackupTempResourceUuid; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotConstant.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotConstant.java index 8f784a72051..adb5c5ea436 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotConstant.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotConstant.java @@ -1,5 +1,10 @@ package org.zstack.header.storage.snapshot; +import org.zstack.header.vm.VmInstanceState; + +import java.util.Arrays; +import java.util.List; + /** */ public interface VolumeSnapshotConstant { @@ -16,4 +21,11 @@ public interface VolumeSnapshotConstant { String VOLUME_SNAPSHOT_STRUCT = "VolumeSnapshotStruct"; String NEED_TAKE_SNAPSHOTS_ON_HYPERVISOR = "needTakeSnapshotOnHypervisor"; String NEED_BLOCK_STREAM_ON_HYPERVISOR = "needBlockStreamOnHypervisor"; + + public static final List ALLOW_TAKE_SNAPSHOTS_VM_STATES = Arrays.asList( + VmInstanceState.Running, VmInstanceState.Stopped, VmInstanceState.Paused + ); + public static final List ALLOW_TAKE_MEMORY_SNAPSHOTS_VM_STATES = Arrays.asList( + VmInstanceState.Stopped + ); } diff --git a/header/src/main/java/org/zstack/header/tpm/RBACInfo.java b/header/src/main/java/org/zstack/header/tpm/RBACInfo.java new file mode 100644 index 00000000000..f0b9e2c82ea --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/RBACInfo.java @@ -0,0 +1,33 @@ +package org.zstack.header.tpm; + +import org.zstack.header.identity.rbac.RBACDescription; +import org.zstack.header.rest.SDKPackage; +import org.zstack.header.tpm.entity.TpmVO; + +@SDKPackage(packageName="org.zstack.sdk.tpm") +public class RBACInfo implements RBACDescription { + @Override + public void permissions() { + permissionBuilder() + .name("tpm") + .normalAPIs("org.zstack.header.tpm.**") + .targetResources(TpmVO.class) + .build(); + } + + @Override + public void contributeToRoles() { + roleContributorBuilder() + .roleName("other") + .actionsByPermissionName("tpm") + .build(); + } + + @Override + public void roles() { + } + + @Override + public void globalReadableResources() { + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/TpmConstants.java b/header/src/main/java/org/zstack/header/tpm/TpmConstants.java new file mode 100644 index 00000000000..4654c5c956d --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/TpmConstants.java @@ -0,0 +1,17 @@ +package org.zstack.header.tpm; + +import org.zstack.header.vm.VmInstanceState; + +import java.util.Collections; +import java.util.List; + +import static org.zstack.utils.CollectionDSL.list; + +public class TpmConstants { + private TpmConstants() {} + + public static final String SERVICE_ID = "tpm"; + + public static final List SUPPORT_VM_STATES_FOR_TPM_OPERATION = + Collections.unmodifiableList(list(VmInstanceState.Stopped)); +} diff --git a/header/src/main/java/org/zstack/header/tpm/TpmErrors.java b/header/src/main/java/org/zstack/header/tpm/TpmErrors.java new file mode 100644 index 00000000000..cb9eebb68c2 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/TpmErrors.java @@ -0,0 +1,22 @@ +package org.zstack.header.tpm; + +public enum TpmErrors { + GENERAL_ERROR(1000), + + // INVALID_ARGUMENT, 17xx <- SYS.1007 + TPM_ALREADY_EXISTS(1701), + TPM_NOT_FOUND(1702), + VM_STATE_ERROR(1703), + ; + + private String code; + + private TpmErrors(int id) { + code = String.format("TPM.%s", id); + } + + @Override + public String toString() { + return code; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmEvent.java b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmEvent.java new file mode 100644 index 00000000000..4106a67afdd --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmEvent.java @@ -0,0 +1,31 @@ +package org.zstack.header.tpm.api; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; +import org.zstack.header.tpm.entity.TpmInventory; + +@RestResponse(allTo = "inventory") +public class APIAddTpmEvent extends APIEvent { + private TpmInventory inventory; + + public APIAddTpmEvent() { + } + + public APIAddTpmEvent(String apiId) { + super(apiId); + } + + public TpmInventory getInventory() { + return inventory; + } + + public void setInventory(TpmInventory inventory) { + this.inventory = inventory; + } + + public static APIAddTpmEvent __example__() { + APIAddTpmEvent event = new APIAddTpmEvent(); + event.setInventory(TpmInventory.__example__()); + return event; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmEventDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..31d912bd097 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmEventDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.entity.TpmInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "虚拟机添加 TPM 的结果" + + ref { + name "inventory" + path "org.zstack.header.tpm.api.APIAddTpmEvent.inventory" + desc "TPM 信息" + type "TpmInventory" + since "5.0.0" + clz TpmInventory.class + } + field { + name "success" + desc "添加是否成功" + type "boolean" + since "5.0.0" + } + ref { + name "error" + path "org.zstack.header.tpm.api.APIAddTpmEvent.error" + desc "错误码，若不为 null，则表示操作失败, 操作成功时该字段为 null" + type "ErrorCode" + since "5.0.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmMsg.java b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmMsg.java new file mode 100644 index 00000000000..3acea51c0fc --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmMsg.java @@ -0,0 +1,51 @@ +package org.zstack.header.tpm.api; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APICreateMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.message.DocUtils; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.vm.VmInstanceMessage; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.utils.StringDSL; + +@RestRequest( + path = "/tpms", + method = HttpMethod.POST, + responseClass = APIAddTpmEvent.class, + parameterName = "params" +) +public class APIAddTpmMsg extends APICreateMessage implements VmInstanceMessage { + /** + * If null, use the default key provider from global config (if set). + */ + @APIParam(required = false, minLength = 32, maxLength = 32) + private String keyProviderUuid; + + @APIParam(resourceType = VmInstanceVO.class) + private String vmInstanceUuid; + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + @Override + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public static APIAddTpmMsg __example__() { + APIAddTpmMsg msg = new APIAddTpmMsg(); + msg.setKeyProviderUuid(StringDSL.createFixedUuid("keyProviderUuid")); + msg.setVmInstanceUuid(DocUtils.createFixedUuid(VmInstanceVO.class)); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..de1e15653ed --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIAddTpmMsgDoc_zh_cn.groovy @@ -0,0 +1,85 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.api.APIAddTpmEvent + +doc { + title "AddTpm" + + category "tpm" + + desc """虚拟机添加 TPM""" + + rest { + request { + url "POST /v1/tpms" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIAddTpmMsg.class + + desc """""" + + params { + + column { + name "keyProviderUuid" + enclosedIn "params" + desc "密钥提供程序 UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "vmInstanceUuid" + enclosedIn "params" + desc "虚拟机 UUID" + location "body" + type "String" + optional false + since "5.0.0" + } + column { + name "resourceUuid" + enclosedIn "params" + desc "资源 UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "tagUuids" + enclosedIn "params" + desc "标签 UUID 列表" + location "body" + type "List" + optional true + since "5.0.0" + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.0.0" + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.0.0" + } + } + } + + response { + clz APIAddTpmEvent.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityMsg.java b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityMsg.java new file mode 100644 index 00000000000..c687bd0e576 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityMsg.java @@ -0,0 +1,48 @@ +package org.zstack.header.tpm.api; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIParam; +import org.zstack.header.message.APISyncCallMessage; +import org.zstack.header.message.DocUtils; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.vm.VmInstanceVO; + +@RestRequest( + path = "/tpms/capability", + method = HttpMethod.GET, + responseClass = APIGetTpmCapabilityReply.class +) +public class APIGetTpmCapabilityMsg extends APISyncCallMessage implements TpmMessage { + @APIParam(required = false, resourceType = TpmVO.class) + private String tpmUuid; + + @APIParam(required = false, resourceType = VmInstanceVO.class) + private String vmInstanceUuid; + + @Override + public String getTpmUuid() { + return tpmUuid; + } + + @Override + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + @Override + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + @Override + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public static APIGetTpmCapabilityMsg __example__() { + APIGetTpmCapabilityMsg msg = new APIGetTpmCapabilityMsg(); + msg.setVmInstanceUuid(DocUtils.createFixedUuid(VmInstanceVO.class)); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..6a232473ad9 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityMsgDoc_zh_cn.groovy @@ -0,0 +1,67 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.api.APIGetTpmCapabilityReply + +doc { + title "GetTpmCapability" + + category "tpm" + + desc """获取 TPM 详情数据""" + + rest { + request { + url "GET /v1/tpms/capability" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIGetTpmCapabilityMsg.class + + desc """""" + + params { + + column { + name "tpmUuid" + enclosedIn "" + desc "TPM UUID" + location "query" + type "String" + optional true + since "5.0.0" + } + column { + name "vmInstanceUuid" + enclosedIn "" + desc "虚拟机 UUID" + location "query" + type "String" + optional true + since "5.0.0" + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "query" + type "List" + optional true + since "5.0.0" + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "query" + type "List" + optional true + since "5.0.0" + } + } + } + + response { + clz APIGetTpmCapabilityReply.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityReply.java b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityReply.java new file mode 100644 index 00000000000..4819ccfc1aa --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityReply.java @@ -0,0 +1,24 @@ +package org.zstack.header.tpm.api; + +import org.zstack.header.message.APIReply; +import org.zstack.header.rest.RestResponse; +import org.zstack.header.tpm.entity.TpmCapabilityView; + +@RestResponse(fieldsTo = "all") +public class APIGetTpmCapabilityReply extends APIReply { + private TpmCapabilityView inventory; + + public TpmCapabilityView getInventory() { + return inventory; + } + + public void setInventory(TpmCapabilityView inventory) { + this.inventory = inventory; + } + + public static APIGetTpmCapabilityReply __example__() { + APIGetTpmCapabilityReply reply = new APIGetTpmCapabilityReply(); + reply.setInventory(TpmCapabilityView.__example__()); + return reply; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityReplyDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..216b1ba86b8 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIGetTpmCapabilityReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.entity.TpmCapabilityView +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "获取 TPM 详情数据的结果" + + ref { + name "inventory" + path "org.zstack.header.tpm.api.APIGetTpmCapabilityReply.inventory" + desc "TPM 性能和信息数据" + type "TpmCapabilityView" + since "5.0.0" + clz TpmCapabilityView.class + } + field { + name "success" + desc "获取是否成功" + type "boolean" + since "5.0.0" + } + ref { + name "error" + path "org.zstack.header.tpm.api.APIGetTpmCapabilityReply.error" + desc "错误码，若不为 null，则表示操作失败, 操作成功时该字段为 null" + type "ErrorCode" + since "5.0.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmMsg.java b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmMsg.java new file mode 100644 index 00000000000..00730943d90 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmMsg.java @@ -0,0 +1,26 @@ +package org.zstack.header.tpm.api; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.DocUtils; +import org.zstack.header.query.APIQueryMessage; +import org.zstack.header.query.AutoQuery; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.tpm.entity.TpmInventory; +import org.zstack.header.tpm.entity.TpmVO; + +import java.util.List; + +import static java.util.Arrays.asList; + +@AutoQuery(replyClass = APIQueryTpmReply.class, inventoryClass = TpmInventory.class) +@RestRequest( + path = "/tpms", + optionalPaths = {"/tpms/{uuid}"}, + method = HttpMethod.GET, + responseClass = APIQueryTpmReply.class +) +public class APIQueryTpmMsg extends APIQueryMessage { + public static List __example__() { + return asList("uuid=" + DocUtils.createFixedUuid(TpmVO.class)); + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..e94cde01431 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmMsgDoc_zh_cn.groovy @@ -0,0 +1,31 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.api.APIQueryTpmReply +import org.zstack.header.query.APIQueryMessage + +doc { + title "QueryTpm" + + category "tpm" + + desc """查询 TPM""" + + rest { + request { + url "GET /v1/tpms" + url "GET /v1/tpms/{uuid}" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIQueryTpmMsg.class + + desc """""" + + params APIQueryMessage.class + } + + response { + clz APIQueryTpmReply.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmReply.java b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmReply.java new file mode 100644 index 00000000000..9262fd88081 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmReply.java @@ -0,0 +1,28 @@ +package org.zstack.header.tpm.api; + +import org.zstack.header.query.APIQueryReply; +import org.zstack.header.rest.RestResponse; +import org.zstack.header.tpm.entity.TpmInventory; + +import java.util.List; + +import static org.zstack.utils.CollectionDSL.list; + +@RestResponse(allTo = "inventories") +public class APIQueryTpmReply extends APIQueryReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } + + public static APIQueryTpmReply __example__() { + APIQueryTpmReply reply = new APIQueryTpmReply(); + reply.setInventories(list(TpmInventory.__example__())); + return reply; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmReplyDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmReplyDoc_zh_cn.groovy new file mode 100644 index 00000000000..bff2c89f924 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIQueryTpmReplyDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.entity.TpmInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "查询 TPM 的结果" + + ref { + name "inventories" + path "org.zstack.header.tpm.api.APIQueryTpmReply.inventories" + desc "TPM 列表" + type "List" + since "5.0.0" + clz TpmInventory.class + } + field { + name "success" + desc "查询是否成功" + type "boolean" + since "5.0.0" + } + ref { + name "error" + path "org.zstack.header.tpm.api.APIQueryTpmReply.error" + desc "错误码，若不为 null，则表示操作失败, 操作成功时该字段为 null" + type "ErrorCode" + since "5.0.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmEvent.java b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmEvent.java new file mode 100644 index 00000000000..7356c2aafa4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmEvent.java @@ -0,0 +1,19 @@ +package org.zstack.header.tpm.api; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; + +@RestResponse +public class APIRemoveTpmEvent extends APIEvent { + public APIRemoveTpmEvent(String apiId) { + super(apiId); + } + + public APIRemoveTpmEvent() { + super(null); + } + + public static APIRemoveTpmEvent __example__() { + return new APIRemoveTpmEvent(); + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmEventDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..bd77dfe1d5f --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmEventDoc_zh_cn.groovy @@ -0,0 +1,23 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "虚拟机删除 TPM 的结果" + + field { + name "success" + desc "删除是否成功" + type "boolean" + since "5.0.0" + } + ref { + name "error" + path "org.zstack.header.tpm.api.APIRemoveTpmEvent.error" + desc "错误码，若不为 null，则表示操作失败, 操作成功时该字段为 null" + type "ErrorCode" + since "5.0.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmMsg.java b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmMsg.java new file mode 100644 index 00000000000..5b5bc148e09 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmMsg.java @@ -0,0 +1,49 @@ +package org.zstack.header.tpm.api; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIDeleteMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.message.DocUtils; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.vm.VmInstanceMessage; +import org.zstack.header.vm.VmInstanceVO; + +@RestRequest( + path = "/tpms", + method = HttpMethod.DELETE, + responseClass = APIRemoveTpmEvent.class +) +public class APIRemoveTpmMsg extends APIDeleteMessage implements VmInstanceMessage, TpmMessage { + @APIParam(required = false, resourceType = TpmVO.class, successIfResourceNotExisting = true) + private String tpmUuid; + + @APIParam(required = false, resourceType = VmInstanceVO.class) + private String vmInstanceUuid; + + @Override + public String getTpmUuid() { + return tpmUuid; + } + + @Override + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + @Override + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + @Override + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public static APIRemoveTpmMsg __example__() { + APIRemoveTpmMsg msg = new APIRemoveTpmMsg(); + msg.setVmInstanceUuid(DocUtils.createFixedUuid(VmInstanceVO.class)); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..0cf9f48131a --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIRemoveTpmMsgDoc_zh_cn.groovy @@ -0,0 +1,76 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.api.APIRemoveTpmEvent + +doc { + title "RemoveTpm" + + category "tpm" + + desc """虚拟机删除 TPM""" + + rest { + request { + url "DELETE /v1/tpms" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIRemoveTpmMsg.class + + desc """""" + + params { + + column { + name "tpmUuid" + enclosedIn "" + desc "TPM UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "vmInstanceUuid" + enclosedIn "" + desc "虚拟机 UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "deleteMode" + enclosedIn "" + desc "删除模式(Permissive / Enforcing，Permissive)" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.0.0" + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.0.0" + } + } + } + + response { + clz APIRemoveTpmEvent.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmEvent.java b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmEvent.java new file mode 100644 index 00000000000..3d03410e053 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmEvent.java @@ -0,0 +1,31 @@ +package org.zstack.header.tpm.api; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; +import org.zstack.header.tpm.entity.TpmInventory; + +@RestResponse(allTo = "inventory") +public class APIUpdateTpmEvent extends APIEvent { + private TpmInventory inventory; + + public APIUpdateTpmEvent() { + } + + public APIUpdateTpmEvent(String apiId) { + super(apiId); + } + + public TpmInventory getInventory() { + return inventory; + } + + public void setInventory(TpmInventory inventory) { + this.inventory = inventory; + } + + public static APIUpdateTpmEvent __example__() { + APIUpdateTpmEvent event = new APIUpdateTpmEvent(); + event.setInventory(TpmInventory.__example__()); + return event; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmEventDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..33f2e7dab69 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmEventDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.entity.TpmInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "更新 TPM 的结果" + + ref { + name "inventory" + path "org.zstack.header.tpm.api.APIUpdateTpmEvent.inventory" + desc "更新后的 TPM 信息" + type "TpmInventory" + since "5.0.0" + clz TpmInventory.class + } + field { + name "success" + desc "更新是否成功" + type "boolean" + since "5.0.0" + } + ref { + name "error" + path "org.zstack.header.tpm.api.APIUpdateTpmEvent.error" + desc "错误码，若不为 null，则表示操作失败, 操作成功时该字段为 null" + type "ErrorCode" + since "5.0.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmMsg.java b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmMsg.java new file mode 100644 index 00000000000..4e3f264d3c3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmMsg.java @@ -0,0 +1,62 @@ +package org.zstack.header.tpm.api; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.message.DocUtils; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.utils.StringDSL; + +@RestRequest( + path = "/tpms", + method = HttpMethod.PUT, + isAction = true, + responseClass = APIUpdateTpmEvent.class +) +public class APIUpdateTpmMsg extends APIMessage implements TpmMessage { + @APIParam(required = false, resourceType = VmInstanceVO.class) + private String vmInstanceUuid; + + @APIParam(required = false, resourceType = TpmVO.class) + private String tpmUuid; + + @APIParam(required = false, minLength = 32, maxLength = 32) + private String keyProviderUuid; + + @Override + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + @Override + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + @Override + public String getTpmUuid() { + return tpmUuid; + } + + @Override + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + public static APIUpdateTpmMsg __example__() { + APIUpdateTpmMsg msg = new APIUpdateTpmMsg(); + msg.setKeyProviderUuid(StringDSL.createFixedUuid("keyProviderUuid")); + msg.setVmInstanceUuid(DocUtils.createFixedUuid(VmInstanceVO.class)); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..56aa1fe0e43 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/APIUpdateTpmMsgDoc_zh_cn.groovy @@ -0,0 +1,76 @@ +package org.zstack.header.tpm.api + +import org.zstack.header.tpm.api.APIUpdateTpmEvent + +doc { + title "UpdateTpm" + + category "tpm" + + desc """更新 TPM""" + + rest { + request { + url "PUT /v1/tpms" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIUpdateTpmMsg.class + + desc """""" + + params { + + column { + name "vmInstanceUuid" + enclosedIn "updateTpm" + desc "虚拟机 UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "tpmUuid" + enclosedIn "updateTpm" + desc "TPM UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "keyProviderUuid" + enclosedIn "updateTpm" + desc "密钥提供程序 UUID" + location "body" + type "String" + optional true + since "5.0.0" + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "5.0.0" + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "5.0.0" + } + } + } + + response { + clz APIUpdateTpmEvent.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/tpm/api/TpmMessage.java b/header/src/main/java/org/zstack/header/tpm/api/TpmMessage.java new file mode 100644 index 00000000000..1ae70c46381 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/api/TpmMessage.java @@ -0,0 +1,8 @@ +package org.zstack.header.tpm.api; + +public interface TpmMessage { + String getVmInstanceUuid(); + void setVmInstanceUuid(String vmInstanceUuid); + String getTpmUuid(); + void setTpmUuid(String tpmUuid); +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmCapabilityView.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmCapabilityView.java new file mode 100644 index 00000000000..3af24b8662c --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmCapabilityView.java @@ -0,0 +1,125 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.configuration.PythonClass; +import org.zstack.header.vm.additions.VmHostFileInventory; + +import java.sql.Timestamp; +import java.util.List; + +import static org.zstack.utils.CollectionDSL.list; + +@PythonClass +public class TpmCapabilityView { + // fields in TpmInventory + private String uuid; + private String name; + private String vmInstanceUuid; + private Timestamp createDate; + private Timestamp lastOpDate; + /** + * collect VmHostFileInventory(VmHostFileVO) type=NvRam or type=TpmState + */ + private List fileRefs; + + // related table fields + // TODO keyProviderUuid / keyProviderType / keyProviderName / keyProviderKeyVersion + + // status fields : from system tags + private String edkVersion; + private String swtpmVersion; + + // config fields : from global / resource config + private boolean resetTpmAfterVmCloneConfig; + + public void setTpmInventory(TpmInventory inventory) { + setUuid(inventory.getUuid()); + setName(inventory.getName()); + setVmInstanceUuid(inventory.getVmInstanceUuid()); + setCreateDate(inventory.getCreateDate()); + setLastOpDate(inventory.getLastOpDate()); + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + public List getFileRefs() { + return fileRefs; + } + + public void setFileRefs(List fileRefs) { + this.fileRefs = fileRefs; + } + + public String getEdkVersion() { + return edkVersion; + } + + public void setEdkVersion(String edkVersion) { + this.edkVersion = edkVersion; + } + + public String getSwtpmVersion() { + return swtpmVersion; + } + + public void setSwtpmVersion(String swtpmVersion) { + this.swtpmVersion = swtpmVersion; + } + + public boolean isResetTpmAfterVmCloneConfig() { + return resetTpmAfterVmCloneConfig; + } + + public void setResetTpmAfterVmCloneConfig(boolean resetTpmAfterVmCloneConfig) { + this.resetTpmAfterVmCloneConfig = resetTpmAfterVmCloneConfig; + } + + public static TpmCapabilityView __example__() { + TpmCapabilityView view = new TpmCapabilityView(); + view.setTpmInventory(TpmInventory.__example__()); + view.setFileRefs(list(VmHostFileInventory.__example__())); + + view.setEdkVersion("edk2-ovmf-20220126gitbb1bba3d77-3.el8.noarch"); + view.setSwtpmVersion("0.8.2"); + + view.setResetTpmAfterVmCloneConfig(true); + return view; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmCapabilityViewDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/entity/TpmCapabilityViewDoc_zh_cn.groovy new file mode 100644 index 00000000000..4ee7ba90b8a --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmCapabilityViewDoc_zh_cn.groovy @@ -0,0 +1,66 @@ +package org.zstack.header.tpm.entity + +import java.sql.Timestamp +import org.zstack.header.vm.additions.VmHostFileInventory + +doc { + + title "TPM 详情" + + field { + name "uuid" + desc "TPM UUID" + type "String" + since "5.0.0" + } + field { + name "name" + desc "TPM 资源名称" + type "String" + since "5.0.0" + } + field { + name "vmInstanceUuid" + desc "虚拟机 UUID" + type "String" + since "5.0.0" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.0.0" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.0.0" + } + ref { + name "fileRefs" + path "org.zstack.header.tpm.entity.TpmCapabilityView.fileRefs" + desc "TPM 相关的主机侧文件或目录数据列表" + type "List" + since "5.0.0" + clz VmHostFileInventory.class + } + field { + name "edkVersion" + desc "EDK 套件版本" + type "String" + since "5.0.0" + } + field { + name "swtpmVersion" + desc "SWTPM 版本" + type "String" + since "5.0.0" + } + field { + name "resetTpmAfterVmCloneConfig" + desc "是否在虚拟机克隆后重置 TPM 状态的配置" + type "boolean" + since "5.0.0" + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmInventory.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmInventory.java new file mode 100644 index 00000000000..eafd210a61d --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmInventory.java @@ -0,0 +1,97 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.configuration.PythonClassInventory; +import org.zstack.header.message.DocUtils; +import org.zstack.header.query.ExpandedQueries; +import org.zstack.header.query.ExpandedQuery; +import org.zstack.header.search.Inventory; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceVO; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Collection; +import java.util.List; + +import static org.zstack.utils.CollectionUtils.transform; + +@PythonClassInventory +@Inventory(mappingVOClass = TpmVO.class) +@ExpandedQueries({ + @ExpandedQuery(expandedField = "vmInstance", inventoryClass = VmInstanceInventory.class, + foreignKey = "vmInstanceUuid", expandedInventoryKey = "uuid"), +}) +public class TpmInventory implements Serializable { + private String uuid; + private String name; + private String vmInstanceUuid; + private Timestamp createDate; + private Timestamp lastOpDate; + + public TpmInventory() { + } + + public static TpmInventory valueOf(TpmVO vo) { + TpmInventory inv = new TpmInventory(); + inv.setUuid(vo.getUuid()); + inv.setName(vo.getResourceName()); + inv.setVmInstanceUuid(vo.getVmInstanceUuid()); + inv.setCreateDate(vo.getCreateDate()); + inv.setLastOpDate(vo.getLastOpDate()); + return inv; + } + + public static List valueOf(Collection vos) { + return transform(vos, TpmInventory::valueOf); + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + public static TpmInventory __example__() { + TpmInventory tpm = new TpmInventory(); + tpm.setUuid(DocUtils.createFixedUuid(TpmVO.class)); + tpm.setVmInstanceUuid(DocUtils.createFixedUuid(VmInstanceVO.class)); + tpm.setName("TPM-for-VM-" + tpm.getVmInstanceUuid()); + tpm.setCreateDate(DocUtils.timestamp()); + tpm.setLastOpDate(DocUtils.timestamp()); + return tpm; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmInventoryDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/tpm/entity/TpmInventoryDoc_zh_cn.groovy new file mode 100644 index 00000000000..f67ac502f79 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmInventoryDoc_zh_cn.groovy @@ -0,0 +1,47 @@ +package org.zstack.header.tpm.entity + +import org.zstack.header.vm.additions.VmHostFileInventory + +doc { + + title "TPM 信息" + + field { + name "uuid" + desc "TPM UUID" + type "String" + since "5.0.0" + } + field { + name "name" + desc "TPM 资源名称" + type "String" + since "5.0.0" + } + field { + name "vmInstanceUuid" + desc "虚拟机 UUID" + type "String" + since "5.0.0" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.0.0" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.0.0" + } + ref { + name "hostRefs" + path "org.zstack.header.tpm.entity.TpmInventory.hostRefs" + desc "TPM 与主机的相关数据列表" + type "List" + since "5.0.0" + clz VmHostFileInventory.class + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmKeyBackupVO.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmKeyBackupVO.java new file mode 100644 index 00000000000..30efe258aa4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmKeyBackupVO.java @@ -0,0 +1,37 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.vo.ResourceVO; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import java.sql.Timestamp; + +/** + * Internal holder for a TPM encryption resource key snapshot during snapshot-group revert. + * The key material is stored in {@code EncryptedResourceKeyRefVO} with {@code resourceUuid} = this VO's uuid. + */ +@Entity +@Table +public class TpmKeyBackupVO extends ResourceVO { + @Column + private Timestamp createDate; + @Column + private Timestamp lastOpDate; + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmKeyBackupVO_.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmKeyBackupVO_.java new file mode 100644 index 00000000000..f2bdc161fcd --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmKeyBackupVO_.java @@ -0,0 +1,13 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.vo.ResourceVO_; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(TpmKeyBackupVO.class) +public class TpmKeyBackupVO_ extends ResourceVO_ { + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmSpec.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmSpec.java new file mode 100644 index 00000000000..1798a775a88 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmSpec.java @@ -0,0 +1,80 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.rest.APINoSee; +import org.zstack.utils.StringDSL; + +public class TpmSpec { + private boolean enable = true; + private String tpmUuid; + private String keyProviderUuid; + @APINoSee + private String secretUuid; + @APINoSee + private String backupFileUuid; + @APINoSee + private boolean resourceKeyCreatedNew; + @APINoSee + private String resourceKeyProviderUuid; + + public boolean isEnable() { + return enable; + } + + public void setEnable(boolean enable) { + this.enable = enable; + } + + public String getTpmUuid() { + return tpmUuid; + } + + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + public String getBackupFileUuid() { + return backupFileUuid; + } + + public void setBackupFileUuid(String backupFileUuid) { + this.backupFileUuid = backupFileUuid; + } + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + + public boolean isResourceKeyCreatedNew() { + return resourceKeyCreatedNew; + } + + public void setResourceKeyCreatedNew(boolean resourceKeyCreatedNew) { + this.resourceKeyCreatedNew = resourceKeyCreatedNew; + } + + public String getResourceKeyProviderUuid() { + return resourceKeyProviderUuid; + } + + public void setResourceKeyProviderUuid(String resourceKeyProviderUuid) { + this.resourceKeyProviderUuid = resourceKeyProviderUuid; + } + + public static TpmSpec __example__() { + TpmSpec tpm = new TpmSpec(); + tpm.setKeyProviderUuid(StringDSL.createFixedUuid("keyProviderUuid")); + return tpm; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmVO.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmVO.java new file mode 100644 index 00000000000..e5fafea5689 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmVO.java @@ -0,0 +1,91 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.identity.OwnedByAccount; +import org.zstack.header.tag.AutoDeleteTag; +import org.zstack.header.vm.VmInstanceEO; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vo.BaseResource; +import org.zstack.header.vo.EntityGraph; +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.ResourceVO; +import org.zstack.header.vo.SoftDeletionCascade; +import org.zstack.header.vo.SoftDeletionCascades; +import org.zstack.header.vo.ToInventory; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import javax.persistence.Transient; +import java.sql.Timestamp; + +@Entity +@Table +@BaseResource +@AutoDeleteTag +@SoftDeletionCascades({ + @SoftDeletionCascade(parent = VmInstanceEO.class, joinColumn = "vmInstanceUuid") +}) +@EntityGraph( + parents = { + @EntityGraph.Neighbour(type = VmInstanceVO.class, myField = "vmInstanceUuid", targetField = "uuid"), + } +) +public class TpmVO extends ResourceVO implements ToInventory, OwnedByAccount { + @Column + @ForeignKey(parentEntityClass = VmInstanceEO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String vmInstanceUuid; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @Transient + private String accountUuid; + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + @Override + public String getAccountUuid() { + return accountUuid; + } + + @Override + public void setAccountUuid(String accountUuid) { + this.accountUuid = accountUuid; + } + + public TpmVO() { + } + + @Override + public String toString() { + return "TpmVO{" + + "vmInstanceUuid='" + vmInstanceUuid + '\'' + + ", uuid='" + uuid + '\'' + + '}'; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/entity/TpmVO_.java b/header/src/main/java/org/zstack/header/tpm/entity/TpmVO_.java new file mode 100644 index 00000000000..99fb5aa3ba5 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/entity/TpmVO_.java @@ -0,0 +1,14 @@ +package org.zstack.header.tpm.entity; + +import org.zstack.header.vo.ResourceVO_; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(TpmVO.class) +public class TpmVO_ extends ResourceVO_ { + public static volatile SingularAttribute vmInstanceUuid; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/AddTpmMsg.java b/header/src/main/java/org/zstack/header/tpm/message/AddTpmMsg.java new file mode 100644 index 00000000000..56696ad4455 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/AddTpmMsg.java @@ -0,0 +1,63 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.tpm.api.APIAddTpmMsg; + +public class AddTpmMsg extends NeedReplyMessage { + /** + * Create new key with the provider uuid. + * "keyProviderUuid" and "resourceUuidKeyFrom" should not be set at the same time. + */ + private String keyProviderUuid; + /** + * Copy key from the resource uuid. + * "keyProviderUuid" and "resourceUuidKeyFrom" should not be set at the same time. + */ + private String resourceUuidKeyFrom; + + private String vmInstanceUuid; + /** + * for creating TpmVO + */ + private String tpmUuid; + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + public String getResourceUuidKeyFrom() { + return resourceUuidKeyFrom; + } + + public void setResourceUuidKeyFrom(String resourceUuidKeyFrom) { + this.resourceUuidKeyFrom = resourceUuidKeyFrom; + } + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public String getTpmUuid() { + return tpmUuid; + } + + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + public static AddTpmMsg valueOf(APIAddTpmMsg api) { + AddTpmMsg msg = new AddTpmMsg(); + msg.setKeyProviderUuid(api.getKeyProviderUuid()); + msg.setVmInstanceUuid(api.getVmInstanceUuid()); + msg.setTpmUuid(api.getResourceUuid()); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/AddTpmReply.java b/header/src/main/java/org/zstack/header/tpm/message/AddTpmReply.java new file mode 100644 index 00000000000..0cbcc9bf1e3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/AddTpmReply.java @@ -0,0 +1,16 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.tpm.entity.TpmInventory; + +public class AddTpmReply extends MessageReply { + private TpmInventory inventory; + + public TpmInventory getInventory() { + return inventory; + } + + public void setInventory(TpmInventory inventory) { + this.inventory = inventory; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/BackupTpmEncryptionKeyMsg.java b/header/src/main/java/org/zstack/header/tpm/message/BackupTpmEncryptionKeyMsg.java new file mode 100644 index 00000000000..e2b0f8eaa4f --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/BackupTpmEncryptionKeyMsg.java @@ -0,0 +1,24 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.NeedReplyMessage; + +public class BackupTpmEncryptionKeyMsg extends NeedReplyMessage { + private String srcResourceUuid; + private String dstResourceUuid; + + public String getSrcResourceUuid() { + return srcResourceUuid; + } + + public void setSrcResourceUuid(String srcResourceUuid) { + this.srcResourceUuid = srcResourceUuid; + } + + public String getDstResourceUuid() { + return dstResourceUuid; + } + + public void setDstResourceUuid(String dstResourceUuid) { + this.dstResourceUuid = dstResourceUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/BackupTpmEncryptionKeyReply.java b/header/src/main/java/org/zstack/header/tpm/message/BackupTpmEncryptionKeyReply.java new file mode 100644 index 00000000000..2ec8bc784a8 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/BackupTpmEncryptionKeyReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.MessageReply; + +public class BackupTpmEncryptionKeyReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/DeleteTpmKeyBackupMsg.java b/header/src/main/java/org/zstack/header/tpm/message/DeleteTpmKeyBackupMsg.java new file mode 100644 index 00000000000..dd512143235 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/DeleteTpmKeyBackupMsg.java @@ -0,0 +1,24 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.NeedReplyMessage; + +public class DeleteTpmKeyBackupMsg extends NeedReplyMessage { + private String tpmUuid; + private String tpmKeyBackupUuid; + + public String getTpmUuid() { + return tpmUuid; + } + + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + public String getTpmKeyBackupUuid() { + return tpmKeyBackupUuid; + } + + public void setTpmKeyBackupUuid(String tpmKeyBackupUuid) { + this.tpmKeyBackupUuid = tpmKeyBackupUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/DeleteTpmKeyBackupReply.java b/header/src/main/java/org/zstack/header/tpm/message/DeleteTpmKeyBackupReply.java new file mode 100644 index 00000000000..aa49fd35dc0 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/DeleteTpmKeyBackupReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.MessageReply; + +public class DeleteTpmKeyBackupReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/RestoreTpmEncryptionKeyMsg.java b/header/src/main/java/org/zstack/header/tpm/message/RestoreTpmEncryptionKeyMsg.java new file mode 100644 index 00000000000..073e652aabe --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/RestoreTpmEncryptionKeyMsg.java @@ -0,0 +1,37 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.NeedReplyMessage; + +public class RestoreTpmEncryptionKeyMsg extends NeedReplyMessage { + private String srcResourceUuid; + private String dstResourceUuid; + /** + * When true, the current encryption key on {@link #dstResourceUuid} (TPM) is copied to a + * {@link org.zstack.header.tpm.entity.TpmKeyBackupVO} before restoring from {@link #srcResourceUuid}. + */ + private boolean backupCurrentKey = true; + + public String getSrcResourceUuid() { + return srcResourceUuid; + } + + public void setSrcResourceUuid(String srcResourceUuid) { + this.srcResourceUuid = srcResourceUuid; + } + + public String getDstResourceUuid() { + return dstResourceUuid; + } + + public void setDstResourceUuid(String dstResourceUuid) { + this.dstResourceUuid = dstResourceUuid; + } + + public boolean isBackupCurrentKey() { + return backupCurrentKey; + } + + public void setBackupCurrentKey(boolean backupCurrentKey) { + this.backupCurrentKey = backupCurrentKey; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/RestoreTpmEncryptionKeyReply.java b/header/src/main/java/org/zstack/header/tpm/message/RestoreTpmEncryptionKeyReply.java new file mode 100644 index 00000000000..fd74241d387 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/RestoreTpmEncryptionKeyReply.java @@ -0,0 +1,15 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.MessageReply; + +public class RestoreTpmEncryptionKeyReply extends MessageReply { + private String tpmKeyBackupUuid; + + public String getTpmKeyBackupUuid() { + return tpmKeyBackupUuid; + } + + public void setTpmKeyBackupUuid(String tpmKeyBackupUuid) { + this.tpmKeyBackupUuid = tpmKeyBackupUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/TpmDeletionMsg.java b/header/src/main/java/org/zstack/header/tpm/message/TpmDeletionMsg.java new file mode 100644 index 00000000000..242cff25353 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/TpmDeletionMsg.java @@ -0,0 +1,34 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.APIDeleteMessage; +import org.zstack.header.message.DeletionMessage; +import org.zstack.header.tpm.api.APIRemoveTpmMsg; + +public class TpmDeletionMsg extends DeletionMessage { + private String tpmUuid; + private String vmInstanceUuid; + + public String getTpmUuid() { + return tpmUuid; + } + + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public static TpmDeletionMsg valueOf(APIRemoveTpmMsg api) { + TpmDeletionMsg msg = new TpmDeletionMsg(); + msg.setTpmUuid(api.getTpmUuid()); + msg.setVmInstanceUuid(api.getVmInstanceUuid()); + msg.setForceDelete(api.getDeletionMode() == APIDeleteMessage.DeletionMode.Enforcing); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/tpm/message/TpmDeletionReply.java b/header/src/main/java/org/zstack/header/tpm/message/TpmDeletionReply.java new file mode 100644 index 00000000000..39e05dc7a24 --- /dev/null +++ b/header/src/main/java/org/zstack/header/tpm/message/TpmDeletionReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.tpm.message; + +import org.zstack.header.message.MessageReply; + +public class TpmDeletionReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java index 6c0155ca41b..1d4e3365316 100644 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java @@ -110,6 +110,9 @@ public class APICreateVmInstanceFromVolumeSnapshotGroupMsg extends APICreateMess @APIParam(required = false) private Map> dataVolumeSystemTags; + @APIParam(required = false) + private Boolean resetTpm; + public String getName() { return name; } @@ -274,4 +277,12 @@ public Map> getDataVolumeSystemTags() { public void setDataVolumeSystemTags(Map> dataVolumeSystemTags) { this.dataVolumeSystemTags = dataVolumeSystemTags; } + + public Boolean getResetTpm() { + return resetTpm; + } + + public void setResetTpm(Boolean resetTpm) { + this.resetTpm = resetTpm; + } } diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsgDoc_zh_cn.groovy index b47385fc846..1b425a7d6fb 100644 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsgDoc_zh_cn.groovy +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsgDoc_zh_cn.groovy @@ -212,6 +212,15 @@ doc { optional true since "4.7.21" } + column { + name "resetTpm" + enclosedIn "params" + desc "是否重置 TPM 状态" + location "body" + type "Boolean" + optional true + since "5.0.0" + } } } diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java index d6c0a4fd7a7..e7fc0e207aa 100755 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java @@ -15,8 +15,10 @@ import org.zstack.header.rest.RestRequest; import org.zstack.header.storage.primary.PrimaryStorageVO; import org.zstack.header.tag.TagResourceType; +import org.zstack.header.vm.devices.VmDevicesSpec; import org.zstack.header.volume.VolumeInventory; import org.zstack.header.zone.ZoneVO; +import org.zstack.utils.gson.JSONObjectUtil; import java.util.Collections; import java.util.List; @@ -324,6 +326,15 @@ public void setName(String name) { @APIParam(required = false) private List diskAOs; + @APIParam(required = false) + private Map devices; + + /** + * cache of {@link #devices} + */ + @APINoSee + private VmDevicesSpec devicesSpec; + public List getDiskAOs() { return diskAOs; } @@ -561,6 +572,26 @@ public void setVirtio(boolean virtio) { this.virtio = virtio; } + public Map getDevices() { + return devices; + } + + public void setDevices(Map devices) { + this.devices = devices; + } + + public VmDevicesSpec getDevicesSpec() { + if (devicesSpec == null && devices != null) { + devicesSpec = JSONObjectUtil.rehashObject(devices, VmDevicesSpec.class); + } + return devicesSpec; + } + + public void setDevicesSpec(VmDevicesSpec devicesSpec) { + this.devicesSpec = devicesSpec; + } + + @SuppressWarnings("unchecked") public static APICreateVmInstanceMsg __example__() { APICreateVmInstanceMsg msg = new APICreateVmInstanceMsg(); msg.setName("vm1"); @@ -572,6 +603,7 @@ public static APICreateVmInstanceMsg __example__() { msg.setL3NetworkUuids(Collections.singletonList(uuid())); msg.setRootVolumeSystemTags(Collections.singletonList("volumeProvisioningStrategy::ThickProvisioning")); msg.setDataVolumeSystemTags(null); + msg.setDevices(JSONObjectUtil.rehashObject(VmDevicesSpec.__example__(), Map.class)); return msg; } diff --git a/header/src/main/java/org/zstack/header/vm/APITakeVmConsoleScreenshotEvent.java b/header/src/main/java/org/zstack/header/vm/APITakeVmConsoleScreenshotEvent.java index b03cad02a3f..a7ee8d9da57 100644 --- a/header/src/main/java/org/zstack/header/vm/APITakeVmConsoleScreenshotEvent.java +++ b/header/src/main/java/org/zstack/header/vm/APITakeVmConsoleScreenshotEvent.java @@ -1,5 +1,6 @@ package org.zstack.header.vm; +import org.zstack.header.log.NoLogging; import org.zstack.header.message.APIEvent; import org.zstack.header.rest.RestResponse; @@ -9,6 +10,7 @@ */ @RestResponse(fieldsTo = {"imageData"}) public class APITakeVmConsoleScreenshotEvent extends APIEvent { + @NoLogging(type = NoLogging.Type.LongText) private String imageData; public APITakeVmConsoleScreenshotEvent() { diff --git a/header/src/main/java/org/zstack/header/vm/ApplianceVmInstanceCreateExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/ApplianceVmInstanceCreateExtensionPoint.java new file mode 100644 index 00000000000..ef723f29a1d --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/ApplianceVmInstanceCreateExtensionPoint.java @@ -0,0 +1,5 @@ +package org.zstack.header.vm; + +public interface ApplianceVmInstanceCreateExtensionPoint { + default void afterPersistApplianceVmInstanceVO(VmInstanceVO vo) {} +} diff --git a/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java index dc364097a8f..3513449cf5a 100755 --- a/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java @@ -1,6 +1,7 @@ package org.zstack.header.vm; import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.vm.devices.VmDevicesSpec; import java.util.ArrayList; import java.util.List; @@ -45,6 +46,7 @@ public class CreateVmInstanceMsg extends NeedReplyMessage implements CreateVmIns private List sshKeyPairUuids; private final List candidatePrimaryStorageUuidsForRootVolume = new ArrayList<>(); private final List candidatePrimaryStorageUuidsForDataVolume = new ArrayList<>(); + private VmDevicesSpec devicesSpec; public List getCandidatePrimaryStorageUuidsForRootVolume() { return candidatePrimaryStorageUuidsForRootVolume; @@ -384,4 +386,11 @@ public boolean getVirtio() { public void setVirtio(boolean virtio) { this.virtio = virtio; } + public VmDevicesSpec getDevicesSpec() { + return devicesSpec; + } + + public void setDevicesSpec(VmDevicesSpec devicesSpec) { + this.devicesSpec = devicesSpec; + } } diff --git a/header/src/main/java/org/zstack/header/vm/InstantiateNewCreatedVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/InstantiateNewCreatedVmInstanceMsg.java index f27bebf9802..a79a5bdab68 100755 --- a/header/src/main/java/org/zstack/header/vm/InstantiateNewCreatedVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/InstantiateNewCreatedVmInstanceMsg.java @@ -1,6 +1,7 @@ package org.zstack.header.vm; import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.vm.devices.VmDevicesSpec; import java.util.ArrayList; import java.util.List; @@ -47,6 +48,7 @@ public void setCandidatePrimaryStorageUuidsForDataVolume(List candidateP } private List diskAOs; + private VmDevicesSpec devicesSpec; public List getDiskAOs() { return diskAOs; @@ -56,6 +58,14 @@ public void setDiskAOs(List diskAOs) { this.diskAOs = diskAOs; } + public VmDevicesSpec getDevicesSpec() { + return devicesSpec; + } + + public void setDevicesSpec(VmDevicesSpec devicesSpec) { + this.devicesSpec = devicesSpec; + } + public List getSoftAvoidHostUuids() { return softAvoidHostUuids; } @@ -197,4 +207,3 @@ public void setDisableL3Networks(List disableL3Networks) { this.disableL3Networks = disableL3Networks; } } - diff --git a/header/src/main/java/org/zstack/header/vm/TakeVmConsoleScreenshotReply.java b/header/src/main/java/org/zstack/header/vm/TakeVmConsoleScreenshotReply.java index 69ad7def25f..b66ceb3e96e 100644 --- a/header/src/main/java/org/zstack/header/vm/TakeVmConsoleScreenshotReply.java +++ b/header/src/main/java/org/zstack/header/vm/TakeVmConsoleScreenshotReply.java @@ -1,5 +1,6 @@ package org.zstack.header.vm; +import org.zstack.header.log.NoLogging; import org.zstack.header.message.MessageReply; /** @@ -7,6 +8,7 @@ * @date 2023-09-11 */ public class TakeVmConsoleScreenshotReply extends MessageReply { + @NoLogging(type = NoLogging.Type.LongText) private String imageData; public String getImageData() { diff --git a/header/src/main/java/org/zstack/header/vm/VmCanonicalEvents.java b/header/src/main/java/org/zstack/header/vm/VmCanonicalEvents.java index 0f87a81fa57..17349128eee 100755 --- a/header/src/main/java/org/zstack/header/vm/VmCanonicalEvents.java +++ b/header/src/main/java/org/zstack/header/vm/VmCanonicalEvents.java @@ -3,6 +3,7 @@ import org.zstack.header.message.NeedJsonSchema; import org.zstack.header.errorcode.ErrorCode; import java.util.Date; +import java.util.List; import java.time.LocalDateTime; /** @@ -21,6 +22,7 @@ public class VmCanonicalEvents { public static final String VM_NIC_INFO_IPRANGE_CONFLICT_PATH = "/vm/nicinfo/iprangeConflict"; public static final String VM_GPU_STATUS_ABNORMAL = "/vm/gpu/status/abnormal"; public static final String VM_GPU_XID_ERROR = "/vm/gpu/xid/error"; + public static final String VM_HOST_FILE_CHANGED_PATH = "/vm/hostfile/changed"; @NeedJsonSchema public static class VmCrashReportData { @@ -399,4 +401,35 @@ public void setMessage(String message) { this.message = message; } } + + @NeedJsonSchema + public static class VmHostFileChangedData { + private String hostUuid; + private String vmUuid; + private List types; + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public List getTypes() { + return types; + } + + public void setTypes(List types) { + this.types = types; + } + } } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java b/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java index ede4cab86cd..e1ab746171a 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java @@ -5,6 +5,7 @@ @PythonClass public interface VmInstanceConstant { String SERVICE_ID = "vmInstance"; + String SECURE_BOOT_SERVICE_ID = "secureBoot"; String ACTION_CATEGORY = "instance"; @PythonClass String USER_VM_TYPE = "UserVm"; @@ -26,6 +27,10 @@ public interface VmInstanceConstant { String L2_TF_VSWITCH_TYPE = "TfL2Network"; String TF_VIRTUAL_NIC_TYPE = "TFVNIC"; + String SHUTDOWN_DETAIL_BY_HOST = "by host"; + String SHUTDOWN_DETAIL_BY_GUEST = "by guest"; + String SHUTDOWN_DETAIL_FINISHED = "finished"; + enum Params { VmInstanceSpec, AttachingVolumeInventory, diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceCreateExtensionPoint.java b/header/src/main/java/org/zstack/header/vm/VmInstanceCreateExtensionPoint.java index c7df865bfa1..a9d21d6b8e2 100644 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceCreateExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceCreateExtensionPoint.java @@ -1,8 +1,19 @@ package org.zstack.header.vm; /** - * Created by lining on 2019/4/17. + * Only for UserVM. + * + * Appliance VM use {@link ApplianceVmInstanceCreateExtensionPoint} */ public interface VmInstanceCreateExtensionPoint { void preCreateVmInstance(CreateVmInstanceMsg msg); + + default void afterPersistVmInstanceVO(VmInstanceVO vo, CreateVmInstanceMsg msg) {} + + /** + * Invoked when VM creation rolls back after + * {`@link` `#afterPersistVmInstanceVO`(VmInstanceVO, CreateVmInstanceMsg)} so extensions can + * clean up any state created in that hook. Implementations should be idempotent. + */ + default void afterRollbackPersistVmInstanceVO(VmInstanceVO vo, CreateVmInstanceMsg msg) {} } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java index 99ee2173b98..aeb947983cb 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceSpec.java @@ -13,6 +13,7 @@ import org.zstack.header.network.l3.L3NetworkInventory; import org.zstack.header.storage.primary.PrimaryStorageInventory; import org.zstack.header.vm.VmInstanceConstant.VmOperation; +import org.zstack.header.vm.devices.VmDevicesSpec; import org.zstack.header.volume.VolumeFormat; import org.zstack.header.volume.VolumeInventory; import org.zstack.header.volume.VolumeType; @@ -405,6 +406,8 @@ public void setCandidatePrimaryStorageUuidsForDataVolume(List candidateP private List disableL3Networks; private List diskAOs; + private VmDevicesSpec devicesSpec = new VmDevicesSpec(); + private OperatingSystemBootingSpec osSpec = new OperatingSystemBootingSpec(); public List getDiskAOs() { return diskAOs; @@ -414,6 +417,22 @@ public void setDiskAOs(List diskAOs) { this.diskAOs = diskAOs; } + public VmDevicesSpec getDevicesSpec() { + return devicesSpec; + } + + public void setDevicesSpec(VmDevicesSpec devicesSpec) { + this.devicesSpec = devicesSpec; + } + + public OperatingSystemBootingSpec getOsSpec() { + return osSpec; + } + + public void setOsSpec(OperatingSystemBootingSpec osSpec) { + this.osSpec = osSpec; + } + public boolean isSkipIpAllocation() { return skipIpAllocation; } @@ -910,4 +929,19 @@ public String getVolumeFormatFromImage() { return ImageConstant.QCOW2_FORMAT_STRING; } } + + public static class OperatingSystemBootingSpec implements Serializable { + /** + * @see VmMachineType + */ + private String machineType; + + public String getMachineType() { + return machineType; + } + + public void setMachineType(String machineType) { + this.machineType = machineType; + } + } } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceVO.java b/header/src/main/java/org/zstack/header/vm/VmInstanceVO.java index 39dc0607e25..62e55080f14 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceVO.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceVO.java @@ -5,6 +5,7 @@ import org.zstack.header.host.HostVO; import org.zstack.header.identity.OwnedByAccount; import org.zstack.header.image.ImageVO; +import org.zstack.header.tpm.entity.TpmVO; import org.zstack.header.vm.cdrom.VmCdRomVO; import org.zstack.header.vo.*; import org.zstack.header.vo.EntityGraph; @@ -35,6 +36,7 @@ @EntityGraph.Neighbour(type = InstanceOfferingVO.class, myField = "instanceOfferingUuid", targetField = "uuid"), @EntityGraph.Neighbour(type = VolumeVO.class, myField = "rootVolumeUuid", targetField = "uuid"), @EntityGraph.Neighbour(type = VmNicVO.class, myField = "uuid", targetField = "vmInstanceUuid"), + @EntityGraph.Neighbour(type = TpmVO.class, myField = "uuid", targetField = "vmInstanceUuid"), } ) public class VmInstanceVO extends VmInstanceAO implements OwnedByAccount, ToInventory { diff --git a/header/src/main/java/org/zstack/header/vm/additions/PackageInfo.java b/header/src/main/java/org/zstack/header/vm/additions/PackageInfo.java new file mode 100644 index 00000000000..dc9921af1f4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/PackageInfo.java @@ -0,0 +1,7 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.rest.SDKPackage; + +@SDKPackage(packageName="org.zstack.sdk.vm.entity") +public class PackageInfo { +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/ResetVmTpmMsg.java b/header/src/main/java/org/zstack/header/vm/additions/ResetVmTpmMsg.java new file mode 100644 index 00000000000..0ce384c46c4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/ResetVmTpmMsg.java @@ -0,0 +1,15 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.NeedReplyMessage; + +public class ResetVmTpmMsg extends NeedReplyMessage { + private String vmInstanceUuid; + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/ResetVmTpmReply.java b/header/src/main/java/org/zstack/header/vm/additions/ResetVmTpmReply.java new file mode 100644 index 00000000000..0f100658fe3 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/ResetVmTpmReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.MessageReply; + +public class ResetVmTpmReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/RestoreVmHostFileMsg.java b/header/src/main/java/org/zstack/header/vm/additions/RestoreVmHostFileMsg.java new file mode 100644 index 00000000000..392e6d12e71 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/RestoreVmHostFileMsg.java @@ -0,0 +1,35 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.vm.VmInstanceMessage; + +public class RestoreVmHostFileMsg extends NeedReplyMessage implements VmInstanceMessage { + private String vmInstanceUuid; + private String snapshotGroupUuid; + private String syncReason; + + @Override + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public String getSnapshotGroupUuid() { + return snapshotGroupUuid; + } + + public void setSnapshotGroupUuid(String snapshotGroupUuid) { + this.snapshotGroupUuid = snapshotGroupUuid; + } + + public String getSyncReason() { + return syncReason; + } + + public void setSyncReason(String syncReason) { + this.syncReason = syncReason; + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/vm/additions/RestoreVmHostFileReply.java b/header/src/main/java/org/zstack/header/vm/additions/RestoreVmHostFileReply.java new file mode 100644 index 00000000000..602059a45dc --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/RestoreVmHostFileReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.MessageReply; + +public class RestoreVmHostFileReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileDeletionMsg.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileDeletionMsg.java new file mode 100644 index 00000000000..b386cc8ecf6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileDeletionMsg.java @@ -0,0 +1,15 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.DeletionMessage; + +public class VmHostBackupFileDeletionMsg extends DeletionMessage { + private String uuid; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileDeletionReply.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileDeletionReply.java new file mode 100644 index 00000000000..b87c233c4c1 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileDeletionReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.MessageReply; + +public class VmHostBackupFileDeletionReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileVO.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileVO.java new file mode 100644 index 00000000000..2c3907cdd65 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileVO.java @@ -0,0 +1,77 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.vo.EntityGraph; +import org.zstack.header.vo.ResourceVO; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Table; +import java.sql.Timestamp; + +/** + * Virtual Machine Host-side File Value Object (Backup files) + * + * Include: NvRam / TpmState files + */ +@Entity +@Table +@EntityGraph( + friends = { + @EntityGraph.Neighbour(type = ResourceVO.class, myField = "resourceUuid", targetField = "uuid"), + } +) +public class VmHostBackupFileVO extends ResourceVO { + @Column + private String resourceUuid; + @Column + @Enumerated(EnumType.STRING) + private VmHostFileType type; + @Column + private Timestamp createDate; + @Column + private Timestamp lastOpDate; + + public String getResourceUuid() { + return resourceUuid; + } + + public void setResourceUuid(String resourceUuid) { + this.resourceUuid = resourceUuid; + } + + public VmHostFileType getType() { + return type; + } + + public void setType(VmHostFileType type) { + this.type = type; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + @Override + public String toString() { + return "VmHostBackupFileVO{" + + "resourceUuid='" + resourceUuid + '\'' + + ", type=" + type + + ", createDate=" + createDate + + ", lastOpDate=" + lastOpDate + + '}'; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileVO_.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileVO_.java new file mode 100644 index 00000000000..355fb0661f9 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostBackupFileVO_.java @@ -0,0 +1,15 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.vo.ResourceVO_; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(VmHostBackupFileVO.class) +public class VmHostBackupFileVO_ extends ResourceVO_ { + public static volatile SingularAttribute resourceUuid; + public static volatile SingularAttribute type; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileBackupJob.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileBackupJob.java new file mode 100644 index 00000000000..4b82fc0b3b8 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileBackupJob.java @@ -0,0 +1,31 @@ +package org.zstack.header.vm.additions; + +public class VmHostFileBackupJob { + private String srcPath; + private String destPath; + private String type; + + public String getSrcPath() { + return srcPath; + } + + public void setSrcPath(String srcPath) { + this.srcPath = srcPath; + } + + public String getDestPath() { + return destPath; + } + + public void setDestPath(String destPath) { + this.destPath = destPath; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentFormat.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentFormat.java new file mode 100644 index 00000000000..1212dc054bc --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentFormat.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm.additions; + +public enum VmHostFileContentFormat { + Raw, + TarballGzip, +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentVO.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentVO.java new file mode 100644 index 00000000000..e08e5a41d70 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentVO.java @@ -0,0 +1,88 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.ResourceVO; +import org.zstack.header.vo.SoftDeletionCascade; +import org.zstack.header.vo.SoftDeletionCascades; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Table; +import java.sql.Timestamp; + +/** + * Virtual Machine Host-side File Content Value Object + */ +@Entity +@Table +@SoftDeletionCascades({ + @SoftDeletionCascade(parent = ResourceVO.class, joinColumn = "uuid"), +}) +public class VmHostFileContentVO { + @Id + @Column + @ForeignKey(parentEntityClass = ResourceVO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String uuid; + @Column + private byte[] content; + @Column + @Enumerated(EnumType.STRING) + private VmHostFileContentFormat format; + @Column + private Timestamp createDate; + @Column + private Timestamp lastOpDate; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public byte[] getContent() { + return content; + } + + public void setContent(byte[] content) { + this.content = content; + } + + public VmHostFileContentFormat getFormat() { + return format; + } + + public void setFormat(VmHostFileContentFormat format) { + this.format = format; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + @Override + public String toString() { + return "VmHostFileContentVO{" + + "uuid='" + uuid + '\'' + + ", format=" + format + + ", createDate=" + createDate + + ", lastOpDate=" + lastOpDate + + '}'; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentVO_.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentVO_.java new file mode 100644 index 00000000000..c375650337b --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileContentVO_.java @@ -0,0 +1,14 @@ +package org.zstack.header.vm.additions; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(VmHostFileContentVO.class) +public class VmHostFileContentVO_ { + public static volatile SingularAttribute uuid; + public static volatile SingularAttribute content; + public static volatile SingularAttribute format; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileDeletionMsg.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileDeletionMsg.java new file mode 100644 index 00000000000..0b4a7499ad1 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileDeletionMsg.java @@ -0,0 +1,15 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.DeletionMessage; + +public class VmHostFileDeletionMsg extends DeletionMessage { + private String uuid; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileDeletionReply.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileDeletionReply.java new file mode 100644 index 00000000000..56f22079798 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileDeletionReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.message.MessageReply; + +public class VmHostFileDeletionReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileInventory.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileInventory.java new file mode 100644 index 00000000000..e229c49024c --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileInventory.java @@ -0,0 +1,139 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.host.HostVO; +import org.zstack.header.message.DocUtils; +import org.zstack.header.vm.VmInstanceVO; + +import java.sql.Timestamp; +import java.util.Collection; +import java.util.List; + +import static org.zstack.utils.CollectionUtils.transform; + +public class VmHostFileInventory { + private String uuid; + private String vmInstanceUuid; + private String hostUuid; + private String type; + private String path; + private String lastSyncReason; + private Timestamp changeDate; + private Timestamp lastSyncDate; + private Timestamp createDate; + private Timestamp lastOpDate; + + public VmHostFileInventory() { + } + + public static VmHostFileInventory valueOf(VmHostFileVO vo) { + VmHostFileInventory inv = new VmHostFileInventory(); + inv.setUuid(vo.getUuid()); + inv.setVmInstanceUuid(vo.getVmInstanceUuid()); + inv.setHostUuid(vo.getHostUuid()); + inv.setType(vo.getType().toString()); + inv.setPath(vo.getPath()); + inv.setLastSyncReason(vo.getLastSyncReason()); + inv.setChangeDate(vo.getChangeDate()); + inv.setLastSyncDate(vo.getLastSyncDate()); + inv.setCreateDate(vo.getCreateDate()); + inv.setLastOpDate(vo.getLastOpDate()); + return inv; + } + + public static List valueOf(Collection vos) { + return transform(vos, VmHostFileInventory::valueOf); + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getLastSyncReason() { + return lastSyncReason; + } + + public void setLastSyncReason(String lastSyncReason) { + this.lastSyncReason = lastSyncReason; + } + + public Timestamp getChangeDate() { + return changeDate; + } + + public void setChangeDate(Timestamp changeDate) { + this.changeDate = changeDate; + } + + public Timestamp getLastSyncDate() { + return lastSyncDate; + } + + public void setLastSyncDate(Timestamp lastSyncDate) { + this.lastSyncDate = lastSyncDate; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + public static VmHostFileInventory __example__() { + VmHostFileInventory ref = new VmHostFileInventory(); + ref.setUuid(DocUtils.createFixedUuid(VmHostFileVO.class)); + ref.setVmInstanceUuid(DocUtils.createFixedUuid(VmInstanceVO.class)); + ref.setHostUuid(DocUtils.createFixedUuid(HostVO.class)); + ref.setType(VmHostFileType.TpmState.toString()); + ref.setPath("/var/lib/libvirt/swtpm/" + ref.getHostUuid() + "/"); + ref.setLastSyncReason("on libvirt shutdown event"); + ref.setCreateDate(DocUtils.timestamp()); + ref.setLastOpDate(DocUtils.timestamp()); + return ref; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileInventoryDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileInventoryDoc_zh_cn.groovy new file mode 100644 index 00000000000..0ec8eb79265 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileInventoryDoc_zh_cn.groovy @@ -0,0 +1,49 @@ +package org.zstack.header.vm.additions + +doc { + + title "虚拟机在主机侧的相关文件或目录数据" + + field { + name "uuid" + desc "相关文件 UUID" + type "String" + since "5.0.0" + } + field { + name "vmInstanceUuid" + desc "虚拟机 UUID" + type "String" + since "5.0.0" + } + field { + name "hostUuid" + desc "主机 UUID" + type "String" + since "5.0.0" + } + field { + name "type" + desc "文件类型, 按用途分类, 可能是 NvRam 或者 TpmState" + type "String" + since "5.0.0" + } + field { + name "path" + desc "主机侧相关文件或目录的路径" + type "String" + since "5.0.0" + } + field { + name "createDate" + desc "创建时间" + type "Timestamp" + since "5.0.0" + } + field { + name "lastOpDate" + desc "最后一次修改时间" + type "Timestamp" + since "5.0.0" + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileManager.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileManager.java new file mode 100644 index 00000000000..342e4fe3d3a --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileManager.java @@ -0,0 +1,5 @@ +package org.zstack.header.vm.additions; + +public interface VmHostFileManager { + void cleanVmHostBackupFile(String resourceUuid); +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileOperation.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileOperation.java new file mode 100644 index 00000000000..8a7be311680 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileOperation.java @@ -0,0 +1,7 @@ +package org.zstack.header.vm.additions; + +public enum VmHostFileOperation { + Write, + Prepare, + Delete, +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileSyncReason.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileSyncReason.java new file mode 100644 index 00000000000..a51ec1a8c67 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileSyncReason.java @@ -0,0 +1,38 @@ +package org.zstack.header.vm.additions; + +public enum VmHostFileSyncReason { + PrepareRead("on prepare host file (from origin host)"), + PrepareReRead("on prepare host file (from dest host)"), + ResourceRelease("on release vm resource"), + PreMigration("on pre-migration (from src host)"), + PostMigration("on post-migration (from dest host)"), + VmShutdown("on libvirt shutdown event"), + PostClone("on post-clone (from dest host)"), + Restore("restore"), + SnapshotGroupOnlineBackup("snapshot group online backup"), + RevertSnapshot("revert snapshot"), + VolumeBackup("volume backup"), + BeforeHaStart("on before HA start (from last host)"), + PeriodicDirtyCheck("on periodic dirty check"), + PeriodicForceSync("on periodic force sync"), + ; + + public final String detail; + + private VmHostFileSyncReason(String detail) { + this.detail = detail; + } + + @Override + public String toString() { + return detail; + } + + public String reason() { + return detail; + } + + public String reason(String description) { + return description == null || description.length() == 0 ? reason() : detail + ": " + description; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileType.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileType.java new file mode 100644 index 00000000000..4416821e949 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileType.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm.additions; + +public enum VmHostFileType { + NvRam, + TpmState, +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileVO.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileVO.java new file mode 100644 index 00000000000..44f3d5423d1 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileVO.java @@ -0,0 +1,140 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.host.HostEO; +import org.zstack.header.host.HostVO; +import org.zstack.header.vm.VmInstanceEO; +import org.zstack.header.vo.EntityGraph; +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.ResourceVO; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Table; +import java.sql.Timestamp; + +/** + * Virtual Machine Host-side File Value Object + * + * Include: NvRam / TpmState files + */ +@Entity +@Table +@EntityGraph( + friends = { + @EntityGraph.Neighbour(type = VmInstanceEO.class, myField = "vmInstanceUuid", targetField = "uuid"), + @EntityGraph.Neighbour(type = HostVO.class, myField = "hostUuid", targetField = "uuid"), + } +) +public class VmHostFileVO extends ResourceVO { + @Column + @ForeignKey(parentEntityClass = VmInstanceEO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String vmInstanceUuid; + @Column + @ForeignKey(parentEntityClass = HostEO.class, onDeleteAction = ForeignKey.ReferenceOption.CASCADE) + private String hostUuid; + @Column + @Enumerated(EnumType.STRING) + private VmHostFileType type; + @Column + private String path; + @Column + private String lastSyncReason; + @Column + private Timestamp changeDate; + @Column + private Timestamp lastSyncDate; + @Column + private Timestamp createDate; + @Column + private Timestamp lastOpDate; + + public String getVmInstanceUuid() { + return vmInstanceUuid; + } + + public void setVmInstanceUuid(String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public VmHostFileType getType() { + return type; + } + + public void setType(VmHostFileType type) { + this.type = type; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getLastSyncReason() { + return lastSyncReason; + } + + public void setLastSyncReason(String lastSyncReason) { + this.lastSyncReason = lastSyncReason; + } + + public Timestamp getChangeDate() { + return changeDate; + } + + public void setChangeDate(Timestamp changeDate) { + this.changeDate = changeDate; + } + + public Timestamp getLastSyncDate() { + return lastSyncDate; + } + + public void setLastSyncDate(Timestamp lastSyncDate) { + this.lastSyncDate = lastSyncDate; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + + @Override + public String toString() { + return "VmHostFileVO{" + + "uuid='" + uuid + '\'' + + ", vmInstanceUuid='" + vmInstanceUuid + '\'' + + ", hostUuid='" + hostUuid + '\'' + + ", type=" + type + + ", path='" + path + '\'' + + ", lastSyncReason='" + lastSyncReason + '\'' + + ", changeDate=" + changeDate + + ", lastSyncDate=" + lastSyncDate + + ", createDate=" + createDate + + ", lastOpDate=" + lastOpDate + + '}'; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/additions/VmHostFileVO_.java b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileVO_.java new file mode 100644 index 00000000000..ab3546aca02 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/additions/VmHostFileVO_.java @@ -0,0 +1,20 @@ +package org.zstack.header.vm.additions; + +import org.zstack.header.vo.ResourceVO_; + +import javax.persistence.metamodel.SingularAttribute; +import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; + +@StaticMetamodel(VmHostFileVO.class) +public class VmHostFileVO_ extends ResourceVO_ { + public static volatile SingularAttribute vmInstanceUuid; + public static volatile SingularAttribute hostUuid; + public static volatile SingularAttribute type; + public static volatile SingularAttribute path; + public static volatile SingularAttribute lastSyncReason; + public static volatile SingularAttribute changeDate; + public static volatile SingularAttribute lastSyncDate; + public static volatile SingularAttribute createDate; + public static volatile SingularAttribute lastOpDate; +} diff --git a/header/src/main/java/org/zstack/header/vm/devices/NvRamSpec.java b/header/src/main/java/org/zstack/header/vm/devices/NvRamSpec.java new file mode 100644 index 00000000000..81eca6330bc --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/devices/NvRamSpec.java @@ -0,0 +1,25 @@ +package org.zstack.header.vm.devices; + +import org.zstack.header.rest.APINoSee; + +public class NvRamSpec { + private boolean needRegister; + @APINoSee + private String backupFileUuid; + + public boolean isNeedRegister() { + return needRegister; + } + + public void setNeedRegister(boolean needRegister) { + this.needRegister = needRegister; + } + + public String getBackupFileUuid() { + return backupFileUuid; + } + + public void setBackupFileUuid(String backupFileUuid) { + this.backupFileUuid = backupFileUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/devices/VmDevicesSpec.java b/header/src/main/java/org/zstack/header/vm/devices/VmDevicesSpec.java new file mode 100644 index 00000000000..11fc43352fa --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/devices/VmDevicesSpec.java @@ -0,0 +1,33 @@ +package org.zstack.header.vm.devices; + +import org.zstack.header.tpm.entity.TpmSpec; + +public class VmDevicesSpec { + /** + * Default value of tpm must be null, because tpm.enable is true by default. + */ + private TpmSpec tpm; + private NvRamSpec nvRam; + + public TpmSpec getTpm() { + return tpm; + } + + public void setTpm(TpmSpec tpm) { + this.tpm = tpm; + } + + public NvRamSpec getNvRam() { + return nvRam; + } + + public void setNvRam(NvRamSpec nvRam) { + this.nvRam = nvRam; + } + + public static VmDevicesSpec __example__() { + VmDevicesSpec spec = new VmDevicesSpec(); + spec.setTpm(TpmSpec.__example__()); + return spec; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/VolumeType.java b/header/src/main/java/org/zstack/header/volume/VolumeType.java index e373e5aef98..6c14a18bd3d 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeType.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeType.java @@ -4,5 +4,5 @@ public enum VolumeType { Root, Data, Memory, - Cache + Cache, } diff --git a/plugin/applianceVm/src/main/java/org/zstack/appliancevm/CreateApplianceVmJob.java b/plugin/applianceVm/src/main/java/org/zstack/appliancevm/CreateApplianceVmJob.java index f25d80f4818..62207f6cd25 100755 --- a/plugin/applianceVm/src/main/java/org/zstack/appliancevm/CreateApplianceVmJob.java +++ b/plugin/applianceVm/src/main/java/org/zstack/appliancevm/CreateApplianceVmJob.java @@ -182,6 +182,10 @@ protected ApplianceVmVO scripts() { } } + final ApplianceVmVO finalVO = avo; + pluginRgty.getExtensionList(ApplianceVmInstanceCreateExtensionPoint.class).forEach( + extension -> extension.afterPersistApplianceVmInstanceVO(finalVO)); + trigger.next(); } diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index 53816200465..a0270c7766d 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -1763,7 +1763,14 @@ private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { cmd.size = msg.getVolume().getSize(); cmd.setShareable(msg.getVolume().isShareable()); cmd.skipIfExisting = msg.isSkipIfExisting(); - cmd.format = msg.hasSystemTag(VolumeSystemTags.FORMAT_QCOW2.getTagFormat()) ? VolumeConstant.VOLUME_FORMAT_QCOW2 : VolumeConstant.VOLUME_FORMAT_RAW ; + + VolumeType type = Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, volumeUuid) + .select(VolumeVO_.type) + .findValue(); + cmd.format = msg.hasSystemTag(VolumeSystemTags.FORMAT_QCOW2.getTagFormat()) ? + VolumeConstant.VOLUME_FORMAT_QCOW2 : + VolumeConstant.VOLUME_FORMAT_RAW ; final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); diff --git a/plugin/kvm/pom.xml b/plugin/kvm/pom.xml index 73e4c81afe3..49594a296e3 100755 --- a/plugin/kvm/pom.xml +++ b/plugin/kvm/pom.xml @@ -2,17 +2,17 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - plugin - org.zstack + plugin + org.zstack 5.5.0 - .. - - kvm - - - org.zstack - compute - ${project.version} + .. + + kvm + + + org.zstack + compute + ${project.version} org.zstack diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/BootKvmStartVmExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/BootKvmStartVmExtension.java new file mode 100644 index 00000000000..9bbc737a1f7 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/BootKvmStartVmExtension.java @@ -0,0 +1,73 @@ +package org.zstack.kvm; + +import org.apache.commons.lang.StringUtils; +import org.zstack.compute.vm.VmSystemTags; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.tag.SystemTagCreator; + +import static org.zstack.kvm.KVMSystemTags.EDK_RPM_TOKEN; +import static org.zstack.kvm.KVMSystemTags.VM_EDK; +import static org.zstack.utils.CollectionDSL.e; +import static org.zstack.utils.CollectionDSL.map; + +/** + * author:kaicai.hu + * Date:2019/12/25 + */ +public class BootKvmStartVmExtension implements KVMStartVmExtensionPoint, KVMSyncVmDeviceInfoExtensionPoint { + + @Override + public void startVmOnKvmSuccess(KVMHostInventory host, VmInstanceSpec spec) { + if (VmSystemTags.BOOT_ORDER_ONCE.hasTag(spec.getVmInventory().getUuid(), VmInstanceVO.class)) { + VmSystemTags.BOOT_ORDER.delete(spec.getVmInventory().getUuid()); + VmSystemTags.BOOT_ORDER_ONCE.delete(spec.getVmInventory().getUuid()); + } + if (VmSystemTags.CDROM_BOOT_ONCE.hasTag(spec.getVmInventory().getUuid(), VmInstanceVO.class)) { + VmSystemTags.BOOT_ORDER.delete(spec.getVmInventory().getUuid()); + VmSystemTags.CDROM_BOOT_ONCE.delete(spec.getVmInventory().getUuid()); + } + + final String machineType = spec.getOsSpec().getMachineType(); + if (StringUtils.isEmpty(machineType)) { + VmSystemTags.MACHINE_TYPE.delete(spec.getVmInventory().getUuid()); + } else { + SystemTagCreator creator = VmSystemTags.MACHINE_TYPE.newSystemTagCreator(spec.getVmInventory().getUuid()); + creator.setTagByTokens(map(e(VmSystemTags.MACHINE_TYPE_TOKEN, machineType))); + creator.inherent = false; + creator.recreate = true; + creator.create(); + } + } + + @Override + public void startVmOnKvmFailed(KVMHostInventory host, VmInstanceSpec spec, ErrorCode err) { + + } + + @Override + public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { + + } + + @Override + public void afterReceiveVmDeviceInfoResponse(VmInstanceInventory vm, KVMAgentCommands.VmDevicesInfoResponse rsp, VmInstanceSpec spec) { + saveVmEdkStatesFromCommand(spec.getVmInventory().getUuid(), rsp); + } + + @SuppressWarnings("unchecked") + private void saveVmEdkStatesFromCommand(String vmUuid, KVMAgentCommands.VmDevicesInfoResponse rsp) { + if (StringUtils.isEmpty(rsp.getEdkRpm())) { + VM_EDK.deleteInherentTag(vmUuid); + return; + } + + SystemTagCreator creator = VM_EDK.newSystemTagCreator(vmUuid); + creator.setTagByTokens(map(e(EDK_RPM_TOKEN, rsp.getEdkRpm()))); + creator.inherent = true; + creator.recreate = true; + creator.create(); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/BootOrderKvmStartVmExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/BootOrderKvmStartVmExtension.java deleted file mode 100644 index d8896ae447b..00000000000 --- a/plugin/kvm/src/main/java/org/zstack/kvm/BootOrderKvmStartVmExtension.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.zstack.kvm; - -import org.zstack.compute.vm.VmSystemTags; -import org.zstack.header.errorcode.ErrorCode; -import org.zstack.header.vm.VmInstanceSpec; -import org.zstack.header.vm.VmInstanceVO; - -/** - * author:kaicai.hu - * Date:2019/12/25 - */ -public class BootOrderKvmStartVmExtension implements KVMStartVmExtensionPoint { - - @Override - public void startVmOnKvmSuccess(KVMHostInventory host, VmInstanceSpec spec) { - if (VmSystemTags.BOOT_ORDER_ONCE.hasTag(spec.getVmInventory().getUuid(), VmInstanceVO.class)) { - VmSystemTags.BOOT_ORDER.delete(spec.getVmInventory().getUuid()); - VmSystemTags.BOOT_ORDER_ONCE.delete(spec.getVmInventory().getUuid()); - } - if (VmSystemTags.CDROM_BOOT_ONCE.hasTag(spec.getVmInventory().getUuid(), VmInstanceVO.class)) { - VmSystemTags.BOOT_ORDER.delete(spec.getVmInventory().getUuid()); - VmSystemTags.CDROM_BOOT_ONCE.delete(spec.getVmInventory().getUuid()); - } - } - - @Override - public void startVmOnKvmFailed(KVMHostInventory host, VmInstanceSpec spec, ErrorCode err) { - - } - - @Override - public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { - - } -} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/HostKeyIdentityHelper.java b/plugin/kvm/src/main/java/org/zstack/kvm/HostKeyIdentityHelper.java new file mode 100644 index 00000000000..d100dbeee17 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/HostKeyIdentityHelper.java @@ -0,0 +1,138 @@ +package org.zstack.kvm; + +import org.apache.commons.lang.StringUtils; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.SimpleQuery; +import org.zstack.core.db.SimpleQuery.Op; +import org.zstack.header.host.HostKeyIdentityVO; +import org.zstack.header.host.HostKeyIdentityVO_; +import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.utils.ExceptionDSL; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.EntityExistsException; +import javax.persistence.PersistenceException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.util.Base64; + +/** + * Shared helper for host envelope key identity: fingerprint and save/update. + * Used by KVMHost (zstack) and KVMEnvelopeKeySyncExtension (premium). + */ +public final class HostKeyIdentityHelper { + private static final CLogger logger = org.zstack.utils.logging.CLoggerImpl.getLogger(HostKeyIdentityHelper.class); + + private HostKeyIdentityHelper() { + } + + /** + * Compute fingerprint from public key (base64): SHA-256 of decoded key bytes, hex-encoded. + * Returns empty string if key is invalid or hashing fails. + */ + public static String fingerprintFromPublicKey(String publicKeyBase64) { + if (publicKeyBase64 == null || publicKeyBase64.isEmpty()) { + return ""; + } + try { + byte[] keyBytes = Base64.getDecoder().decode(publicKeyBase64.trim()); + if (keyBytes.length == 0) { + return ""; + } + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] hash = md.digest(keyBytes); + StringBuilder sb = new StringBuilder(hash.length * 2); + for (byte b : hash) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } catch (IllegalArgumentException | NoSuchAlgorithmException e) { + return ""; + } + } + + public static HostKeyIdentityVO getHostKeyIdentity(DatabaseFacade dbf, String hostUuid) { + SimpleQuery q = dbf.createQuery(HostKeyIdentityVO.class); + q.add(HostKeyIdentityVO_.hostUuid, Op.EQ, hostUuid); + return q.find(); + } + + /** + * Save or update host key identity. Validates publicKey (base64, 32 bytes, non-empty fingerprint). + * On invalid key, marks existing record as verified=false and returns without persisting bad data. + */ + public static void saveOrUpdateHostKeyIdentity(DatabaseFacade dbf, String hostUuid, String publicKey, boolean verified) { + if (StringUtils.isBlank(publicKey)) { + return; + } + + String keyToSave = publicKey.trim(); + byte[] decodedKey; + try { + decodedKey = Base64.getDecoder().decode(keyToSave); + } catch (IllegalArgumentException e) { + logger.warn("host " + hostUuid + " returned an invalid envelope public key"); + setVerified(dbf, hostUuid, false); + return; + } + if (decodedKey.length != 32) { + logger.warn("host " + hostUuid + " returned an envelope public key with unexpected length: " + decodedKey.length); + setVerified(dbf, hostUuid, false); + return; + } + String fingerprint = fingerprintFromPublicKey(keyToSave); + if (StringUtils.isBlank(fingerprint)) { + logger.warn("host " + hostUuid + " returned an envelope public key with empty fingerprint"); + setVerified(dbf, hostUuid, false); + return; + } + + HostKeyIdentityVO vo = getHostKeyIdentity(dbf, hostUuid); + if (vo == null) { + vo = new HostKeyIdentityVO(); + vo.setHostUuid(hostUuid); + vo.setPublicKey(keyToSave); + vo.setFingerprint(fingerprint); + vo.setVerified(verified); + vo.setCreateDate(new java.sql.Timestamp(System.currentTimeMillis())); + try { + dbf.persist(vo); + return; + } catch (PersistenceException e) { + if (!ExceptionDSL.isCausedBy(e, EntityExistsException.class) + && !ExceptionDSL.isCausedBy(e, SQLIntegrityConstraintViolationException.class, "Duplicate entry")) { + throw e; + } + vo = getHostKeyIdentity(dbf, hostUuid); + if (vo == null) { + throw e; + } + } + } + vo.setPublicKey(keyToSave); + vo.setFingerprint(fingerprint); + vo.setVerified(verified); + dbf.update(vo); + } + + /** + * Set verified flag for existing host key identity record, if present. + */ + public static void setVerified(DatabaseFacade dbf, String hostUuid, boolean verified) { + HostKeyIdentityVO vo = getHostKeyIdentity(dbf, hostUuid); + if (vo != null) { + vo.setVerified(verified); + dbf.update(vo); + } + } + + /** + * Whether the verify response error code indicates that key rotate/re-fetch is needed. + */ + public static boolean isRotateNeededGetError(String errorCode) { + if (errorCode == null) return false; + return SecretHostDefineReply.ERROR_CODE_KEYS_NOT_ON_DISK.equals(errorCode) + || SecretHostDefineReply.ERROR_CODE_KEY_FILES_INTEGRITY_MISMATCH.equals(errorCode); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/HostSecretEnvelopeCryptoExtensionPoint.java b/plugin/kvm/src/main/java/org/zstack/kvm/HostSecretEnvelopeCryptoExtensionPoint.java new file mode 100644 index 00000000000..252bffbf055 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/HostSecretEnvelopeCryptoExtensionPoint.java @@ -0,0 +1,13 @@ +package org.zstack.kvm; + +/** + * Extension point for sealing plaintext (e.g. DEK) with recipient's X25519 public key for host secret (e.g. vTPM). + * Implementation is provided by premium/crypto when available; KVM uses PluginRegistry.getExtensionList() to get it. + */ +public interface HostSecretEnvelopeCryptoExtensionPoint { + /** + * Seal plaintext with recipient's X25519 public key (raw 32 bytes). + * @return envelope = enc (32 bytes) || ciphertext (AEAD output), compatible with key-agent open. + */ + byte[] seal(byte[] recipientPublicKey, byte[] plaintext) throws Exception; +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index ab2aeda3c09..7e3282a9648 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -13,8 +13,10 @@ import org.zstack.header.host.VmNicRedirectConfig; import org.zstack.header.log.NoLogging; import org.zstack.header.vm.*; +import org.zstack.header.vm.additions.VmHostFileBackupJob; import org.zstack.header.vm.devices.DeviceAddress; import org.zstack.header.vm.devices.VirtualDeviceInfo; +import org.zstack.kvm.tpm.TpmTO; import org.zstack.network.securitygroup.RuleTO; import org.zstack.network.securitygroup.SecurityGroupMembersTO; import org.zstack.network.securitygroup.VmNicSecurityTO; @@ -23,6 +25,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.zstack.utils.CollectionUtils.transform; + public class KVMAgentCommands { public enum BootDev { hd(VmBootDevice.HardDisk), @@ -398,6 +402,235 @@ public void setAgentStartTimeMillis(long agentStartTimeMillis) { } } + public static class CreatePublicKeyCmd extends AgentCommand { + } + + public static class CreatePublicKeyResponse extends AgentResponse { + } + + public static class GetPublicKeyCmd extends AgentCommand { + } + + public static class GetPublicKeyResponse extends AgentResponse { + private String publicKey; + + public String getPublicKey() { + return publicKey; + } + + public void setPublicKey(String publicKey) { + this.publicKey = publicKey; + } + } + + public static class RotatePublicKeyCmd extends AgentCommand { + } + + public static class RotatePublicKeyResponse extends AgentResponse { + } + + public static class VerifyPublicKeyCmd extends AgentCommand { + } + + public static class VerifyPublicKeyResponse extends AgentResponse { + } + + public static class SecretHostDefineCmd extends AgentCommand { + /** Base64 envelope of DEK; agent expects this field name (encryptedDek). */ + private String encryptedDek; + private String vmUuid; + private String purpose; + private Integer keyVersion; + private String usageInstance; + private String secretUuid; + private String description; + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getUsageInstance() { + return usageInstance; + } + + public void setUsageInstance(String usageInstance) { + this.usageInstance = usageInstance; + } + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + } + + public static class SecretHostDefineResponse extends AgentResponse { + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + } + + public static class SecretHostGetCmd extends AgentCommand { + private String vmUuid; + private String purpose; + private Integer keyVersion; + private String usageInstance; + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getUsageInstance() { + return usageInstance; + } + + public void setUsageInstance(String usageInstance) { + this.usageInstance = usageInstance; + } + } + + public static class SecretHostGetResponse extends AgentResponse { + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + } + + public static class ResolveVtpmLibvirtSecretCmd extends AgentCommand { + private String vmUuid; + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + } + + public static class ResolveVtpmLibvirtSecretResponse extends AgentResponse { + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + } + + public static class SecretHostDeleteCmd extends AgentCommand { + private String vmUuid; + private String purpose; + private Integer keyVersion; + private String usageInstance; + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public String getUsageInstance() { + return usageInstance; + } + + public void setUsageInstance(String usageInstance) { + this.usageInstance = usageInstance; + } + } + + public static class SecretHostDeleteResponse extends AgentResponse { + } + public static class PingCmd extends AgentCommand { public String hostUuid; public long kvmagentPhysicalMemoryUsageAlarmThreshold; @@ -2351,7 +2584,9 @@ public static class StartVmCmd extends vdiCmd implements VmAddOnsCmd { private List dataVolumes; @GrayVersion(value = "5.0.0") private List cacheVolumes; + private VolumeTO nvRam; private List nics; + private TpmTO tpm; @GrayVersion(value = "5.0.0") private long timeout; @GrayVersion(value = "5.0.0") @@ -2418,6 +2653,8 @@ public static class StartVmCmd extends vdiCmd implements VmAddOnsCmd { // used when bootMode == 'UEFI' @GrayVersion(value = "5.0.0") private boolean secureBoot; + private String edkVersion; + @GrayVersion(value = "5.0.0") private boolean fromForeignHypervisor; @GrayVersion(value = "5.0.0") @@ -2584,6 +2821,14 @@ public void setSecureBoot(boolean secureBoot) { this.secureBoot = secureBoot; } + public String getEdkVersion() { + return edkVersion; + } + + public void setEdkVersion(String edkVersion) { + this.edkVersion = edkVersion; + } + public boolean isEmulateHyperV() { return emulateHyperV; } @@ -2896,6 +3141,14 @@ public void setCacheVolumes(List cacheVolumes) { this.cacheVolumes = cacheVolumes; } + public VolumeTO getNvRam() { + return nvRam; + } + + public void setNvRam(VolumeTO nvRam) { + this.nvRam = nvRam; + } + public List getNics() { return nics; } @@ -2904,6 +3157,14 @@ public void setNics(List nics) { this.nics = nics; } + public TpmTO getTpm() { + return tpm; + } + + public void setTpm(TpmTO tpm) { + this.tpm = tpm; + } + public long getTimeout() { return timeout; } @@ -3081,6 +3342,80 @@ public void setHostMinimumFreeMemorySize(Long hostMinimumFreeMemorySize) { public static class StartVmResponse extends VmDevicesInfoResponse { } + public static class VmHostFileTO { + private String path; + /** + * maybe "NvRam" or "TpmState" ... + * @see org.zstack.header.vm.additions.VmHostFileType + */ + private String type; + /** + * file format: Raw or TarballGzip + * @see org.zstack.header.vm.additions.VmHostFileContentFormat + */ + private String fileFormat; + /** + * operation: Write, Prepare, or Delete + * only use in WriteVmHostFileContentCmd + * @see org.zstack.header.vm.additions.VmHostFileOperation + */ + private String operation; + /** + * null if operation is Prepare or Delete + */ + @NoLogging(type = NoLogging.Type.LongText) + private String contentBase64; + private String error; + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getFileFormat() { + return fileFormat; + } + + public void setFileFormat(String fileFormat) { + this.fileFormat = fileFormat; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + + public String getContentBase64() { + return contentBase64; + } + + public void setContentBase64(String contentBase64) { + this.contentBase64 = contentBase64; + } + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + } + } + public static class VmDevicesInfoResponse extends AgentResponse { @GrayVersion(value = "5.0.0") private List nicInfos; @@ -3094,6 +3429,7 @@ public static class VmDevicesInfoResponse extends AgentResponse { private Map pciDeviceInfos; @GrayVersion(value = "5.5.6") private Map mdevDeviceInfos; + private String edkRpm; public VirtualDeviceInfo getMemBalloonInfo() { return memBalloonInfo; @@ -3127,6 +3463,14 @@ public void setVirtualizerInfo(VirtualizerInfoTO virtualizerInfo) { this.virtualizerInfo = virtualizerInfo; } + public String getEdkRpm() { + return edkRpm; + } + + public void setEdkRpm(String edkRpm) { + this.edkRpm = edkRpm; + } + public Map getPciDeviceInfos() { return pciDeviceInfos; } @@ -3160,6 +3504,67 @@ public void setVmInstanceUuid(String vmInstanceUuid) { public static class SyncVmDeviceInfoResponse extends VmDevicesInfoResponse { } + public static class ReadVmHostFileContentCmd extends AgentCommand { + /** + * without contentBase64, fileFormat + */ + private List hostFiles = new ArrayList<>(); + + public List getPaths() { + return transform(hostFiles, VmHostFileTO::getPath); + } + + public List getHostFiles() { + return hostFiles; + } + + public void setHostFiles(List hostFiles) { + this.hostFiles = hostFiles; + } + } + + public static class ReadVmHostFileContentResponse extends AgentResponse { + private List hostFiles = new ArrayList<>(); + + public List getHostFiles() { + return hostFiles; + } + + public void setHostFiles(List hostFiles) { + this.hostFiles = hostFiles; + } + } + + public static class WriteVmHostFileContentCmd extends AgentCommand { + private List hostFiles = new ArrayList<>(); + + public List getHostFiles() { + return hostFiles; + } + + public void setHostFiles(List hostFiles) { + this.hostFiles = hostFiles; + } + } + + public static class WriteVmHostFileContentResponse extends AgentResponse { + } + + public static class BackupVmHostFileCmd extends AgentCommand { + private List vmHostFileBackupJobs; + + public List getVmHostFileBackupJobs() { + return vmHostFileBackupJobs; + } + + public void setVmHostFileBackupJobs(List vmHostFileBackupJobs) { + this.vmHostFileBackupJobs = vmHostFileBackupJobs; + } + } + + public static class BackupVmHostFileResponse extends AgentResponse { + } + public static class VmNicInfo { private String macAddress; private DeviceAddress deviceAddress; @@ -4801,6 +5206,12 @@ public static class ReportHostStopEventCmd { public String hostUuid; } + public static class ReportVmHostFileChangedCmd { + public String hostUuid; + public String vmUuid; + public List types; + } + public static class ShutdownHostCmd extends AgentCommand { } @@ -5015,6 +5426,7 @@ public void setVmUuid(String vmUuid) { public static class TakeVmConsoleScreenshotRsp extends AgentResponse { @GrayVersion(value = "5.0.0") + @NoLogging(type = NoLogging.Type.LongText) private String imageData; public String getImageData() { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java index 4ae21dac461..c8ba44fd703 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java @@ -1,7 +1,9 @@ package org.zstack.kvm; import org.zstack.header.configuration.PythonClass; +import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.additions.VmHostFileType; @PythonClass public interface KVMConstant { @@ -99,6 +101,10 @@ public interface KVMConstant { String KVM_REGISTER_PRIMARY_VM_HEARTBEAT = "/register/primary/vm/heartbeat"; String CLEAN_FIRMWARE_FLASH = "/clean/firmware/flash"; String FSTRIM_VM_PATH = "/vm/fstrim"; + String READ_VM_HOST_FILE_PATH = "/vm/hostfile/read"; + String WRITE_VM_HOST_FILE_PATH = "/vm/hostfile/write"; + String BACKUP_VM_HOST_FILE_PATH = "/vm/hostfile/backup"; + String KVM_VTPM_RESOLVE_LIBVIRT_SECRET_UUID_PATH = "/vm/vtpm/resolveLibvirtSecretUuid"; // ZSTAC-83157: virtiofs model mount paths String KVM_VIRTIOFS_ATTACH_PATH = "/virtiofs/attach"; @@ -131,6 +137,7 @@ public interface KVMConstant { String KVM_REPORT_VM_CRASH_EVENT = "/kvm/reportvmcrash"; String KVM_REPORT_VM_START_EVENT = "/kvm/reportvmstart"; String KVM_REPORT_HOST_STOP_EVENT = "/kvm/reporthoststop"; + String KVM_REPORT_VM_HOST_FILE_CHANGED = "/kvm/reporthostfilechanged"; String KVM_TRANSMIT_VM_OPERATION_TO_MN = "/host/transmitvmoperation"; String KVM_HOST_PHYSICAL_NIC_ALARM_EVENT = "/host/physicalNic/alarm"; @@ -144,6 +151,25 @@ public interface KVMConstant { String RECONCILE_VM_HA_ENABLED_METADATA_LIVE_PATH = "/host/vm/reconcileHaEnabledMetadata/live"; String HA_NETWORK_GROUP_SYNC_PATH = "/ha/networkgroup/sync"; + String KVM_CREATE_ENVELOPE_KEY_PATH = "/host/key/envelope/createEnvelopeKey"; + String KVM_GET_ENVELOPE_KEY_PATH = "/host/key/envelope/getEnvelopePublicKey"; + String KVM_ROTATE_ENVELOPE_KEY_PATH = "/host/key/envelope/rotateEnvelopeKey"; + String KVM_VERIFY_ENVELOPE_KEY_PATH = "/host/key/envelope/checkEnvelopeKey"; + String KVM_GET_SECRET_PATH = "/host/key/envelope/getSecret"; + String KVM_ENSURE_SECRET_PATH = "/host/key/envelope/ensureSecret"; + String KVM_DELETE_SECRET_PATH = "/host/key/envelope/deleteSecret"; + + /** HTTP timeout in seconds for envelope key sync (verify/create/rotate/get) to agent. */ + long ENVELOPE_KEY_HTTP_TIMEOUT_SEC = 10L; + + /** Max size in bytes for DEK payload in SecretHostDefine (decoded from dekBase64). */ + int MAX_DEK_BYTES = 1024; + String HOST_SECRET_USAGE_INSTANCE_VTPM = "tpm0"; + + String KVM_HOST_FILE_DOWNLOAD_PATH = "/host/file/download"; + String KVM_HOST_FILE_UPLOAD_PATH = "/host/file/upload"; + String KVM_HOST_FILE_DOWNLOAD_PROGRESS_PATH = "/host/file/progress"; + String KVM_HOST_IPSET_ATTACH_NIC_PATH = "/network/ipset/attach"; String KVM_HOST_IPSET_DETACH_NIC_PATH = "/network/ipset/detach"; String KVM_HOST_IPSET_SYNC_PATH = "/network/ipset/sync"; @@ -204,6 +230,45 @@ public interface KVMConstant { public static final String L2_PROVIDER_TYPE_LINUX_BRIDGE = "LinuxBridge"; + public static final String EDK_VERSION_NONE = "None"; + public static final String NV_RAM_FILE_PATH_FORMAT = "/var/lib/libvirt/qemu/nvram/%s-host-files/%s.fd"; + public static String buildNvramFilePath(String vmUuid) { + return String.format(NV_RAM_FILE_PATH_FORMAT, vmUuid, vmUuid); + } + + public static final String TPM_STATE_FILE_PATH_FORMAT = "/var/lib/libvirt/swtpm/%s/"; + public static String buildTpmStateFilePath(String vmUuid) { + String vmUuidWithHyphen = vmUuid.replaceFirst("(\\w{8})(\\w{4})(\\w{4})(\\w{4})(\\w{12})", "$1-$2-$3-$4-$5"); + return String.format(TPM_STATE_FILE_PATH_FORMAT, vmUuidWithHyphen); + } + + public static String buildPathForVmHostFileType(VmHostFileType type, String vmUuid) { + switch (type) { + case NvRam: return buildNvramFilePath(vmUuid); + case TpmState: return buildTpmStateFilePath(vmUuid); + default: throw new CloudRuntimeException("unsupported VmHostFileType: " + type); + } + } + + public static final String NV_RAM_SNAPSHOT_BACKUP_FILE_PATH_FORMAT = "/var/lib/libvirt/qemu/nvram/%s-host-files/%s.fd.snapshot-backup"; + public static String buildNvramSnapshotBackupFilePath(String vmUuid) { + return String.format(NV_RAM_SNAPSHOT_BACKUP_FILE_PATH_FORMAT, vmUuid, vmUuid); + } + + public static final String TPM_STATE_SNAPSHOT_BACKUP_FILE_PATH_FORMAT = "/var/lib/libvirt/swtpm/%s.snapshot-backup/"; + public static String buildTpmStateSnapshotBackupFilePath(String vmUuid) { + String vmUuidWithHyphen = vmUuid.replaceFirst("(\\w{8})(\\w{4})(\\w{4})(\\w{4})(\\w{12})", "$1-$2-$3-$4-$5"); + return String.format(TPM_STATE_SNAPSHOT_BACKUP_FILE_PATH_FORMAT, vmUuidWithHyphen); + } + + public static String buildSnapshotBackupPathForVmHostFileType(VmHostFileType type, String vmUuid) { + switch (type) { + case NvRam: return buildNvramSnapshotBackupFilePath(vmUuid); + case TpmState: return buildTpmStateSnapshotBackupFilePath(vmUuid); + default: throw new CloudRuntimeException("unsupported VmHostFileType: " + type); + } + } + public static final String DHCP_BIN_FILE_PATH = "/usr/local/zstack/dnsmasq"; String KVM_HOST_NETWORK_INTERFACE_DEFAULT = "None"; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index e76c271e879..ee8e80346da 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -11,6 +11,8 @@ import org.zstack.header.host.HostVO; import org.zstack.header.zone.ZoneVO; +import static org.zstack.kvm.KVMConstant.EDK_VERSION_NONE; + /** */ @GlobalConfigDefinition @@ -166,4 +168,18 @@ public class KVMGlobalConfig { @BindResourceConfig({HostVO.class, ClusterVO.class}) public static GlobalConfig MINIMUM_MEMORY_SIZE_BEFORE_START_VM = new GlobalConfig(CATEGORY, "min.free.memory.size"); + @GlobalConfigDef(defaultValue = EDK_VERSION_NONE, + description = "Specify the EDK version to be used for the next VM startup. None indicates the use of the system's default EDK version") + @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class}) + public static GlobalConfig VM_EDK_VERSION_CONFIG = new GlobalConfig(CATEGORY, "vm.edk.version"); + + @GlobalConfigValidation(numberGreaterThan = 1, numberLessThan = 86400) + @GlobalConfigDef(defaultValue = "15", type = Long.class, + description = "Interval in seconds for checking VM host files (NvRam, TpmState) on KVM hosts") + public static GlobalConfig VM_HOST_FILE_SYNC_INTERVAL = new GlobalConfig(CATEGORY, "vm.host.file.sync.interval"); + + @GlobalConfigValidation(numberGreaterThan = 1, numberLessThan = 30) + @GlobalConfigDef(defaultValue = "5", type = Integer.class, + description = "The concurrency level for syncing VM host files from KVM hosts") + public static GlobalConfig VM_HOST_FILE_SYNC_CONCURRENCY = new GlobalConfig(CATEGORY, "vm.host.file.sync.concurrency"); } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index b7f7169918f..ebb7e03866c 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -65,6 +65,14 @@ import org.zstack.header.host.*; import org.zstack.header.host.MigrateVmOnHypervisorMsg.StorageMigrationPolicy; import org.zstack.header.image.*; +import org.zstack.header.secret.SecretHostDefineMsg; +import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.header.secret.SecretHostDeleteMsg; +import org.zstack.header.secret.SecretHostDeleteReply; +import org.zstack.header.secret.SecretHostGetMsg; +import org.zstack.header.secret.SecretHostGetReply; +import org.zstack.header.secret.ResolveVtpmLibvirtSecretOnHypervisorMsg; +import org.zstack.header.secret.ResolveVtpmLibvirtSecretOnHypervisorReply; import org.zstack.header.message.APIMessage; import org.zstack.header.message.Message; import org.zstack.header.message.MessageReply; @@ -313,6 +321,8 @@ public static Set parseSanIps(String sanOutput) { private String blockPullPath; private String agentPackageName = KVMGlobalProperty.AGENT_PACKAGE_NAME; private String hostTakeOverFlagPath = KVMGlobalProperty.TAKEVOERFLAGPATH; + private String readVmHostFilePath; + private String writeVmHostFilePath; public KVMHost(KVMHostVO self, KVMHostContext context) { super(self); @@ -543,6 +553,14 @@ public KVMHost(KVMHostVO self, KVMHostContext context) { ub = UriComponentsBuilder.fromHttpUrl(baseUrl); ub.path(KVMConstant.KVM_BLOCK_PULL_VOLUME_PATH); blockPullPath = ub.build().toString(); + + ub = UriComponentsBuilder.fromHttpUrl(baseUrl); + ub.path(KVMConstant.READ_VM_HOST_FILE_PATH); + readVmHostFilePath = ub.build().toString(); + + ub = UriComponentsBuilder.fromHttpUrl(baseUrl); + ub.path(KVMConstant.WRITE_VM_HOST_FILE_PATH); + writeVmHostFilePath = ub.build().toString(); } static { @@ -805,6 +823,14 @@ protected void handleLocalMessage(Message msg) { handle((UpdateVmConsolePasswordOnHypervisorMsg) msg); } else if (msg instanceof GetBlockDevicesOnHostMsg) { handle((GetBlockDevicesOnHostMsg) msg); + } else if (msg instanceof SecretHostGetMsg) { + handle((SecretHostGetMsg) msg); + } else if (msg instanceof ResolveVtpmLibvirtSecretOnHypervisorMsg) { + handle((ResolveVtpmLibvirtSecretOnHypervisorMsg) msg); + } else if (msg instanceof SecretHostDefineMsg) { + handle((SecretHostDefineMsg) msg); + } else if (msg instanceof SecretHostDeleteMsg) { + handle((SecretHostDeleteMsg) msg); } else { super.handleLocalMessage(msg); } @@ -4691,9 +4717,15 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi cmd.setChassisAssetTag(NetworkGlobalProperty.CHASSIS_ASSET_TAG); } - String machineType = VmSystemTags.MACHINE_TYPE.getTokenByResourceUuid(cmd.getVmInstanceUuid(), - VmInstanceVO.class, VmSystemTags.MACHINE_TYPE_TOKEN); - cmd.setMachineType(StringUtils.isNotEmpty(machineType) ? machineType : "pc"); + if (spec.getOsSpec().getMachineType() == null) { + String machineType = VmSystemTags.MACHINE_TYPE.getTokenByResourceUuid(cmd.getVmInstanceUuid(), + VmInstanceVO.class, VmSystemTags.MACHINE_TYPE_TOKEN); + cmd.setMachineType(StringUtils.isNotEmpty(machineType) ? machineType : "pc"); + spec.getOsSpec().setMachineType(cmd.getMachineType()); + } else { + cmd.setMachineType(spec.getOsSpec().getMachineType()); + } + String machineType = cmd.getMachineType(); if (KVMSystemTags.VM_PREDEFINED_PCI_BRIDGE_NUM.hasTag(spec.getVmInventory().getUuid())) { cmd.setPredefinedPciBridgeNum(Integer.valueOf(KVMSystemTags.VM_PREDEFINED_PCI_BRIDGE_NUM.getTokenByResourceUuid(spec.getVmInventory().getUuid(), KVMSystemTags.VM_PREDEFINED_PCI_BRIDGE_NUM_TOKEN))); @@ -4802,10 +4834,6 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi String bootMode = VmSystemTags.BOOT_MODE.getTokenByResourceUuid(spec.getVmInventory().getUuid(), VmSystemTags.BOOT_MODE_TOKEN); cmd.setBootMode(bootMode == null ? ImageBootMode.Legacy.toString() : bootMode); - if (cmd.getBootMode().equals(ImageBootMode.UEFI.toString()) - || cmd.getBootMode().equals(ImageBootMode.UEFI_WITH_CSM.toString())) { - cmd.setSecureBoot(VmGlobalConfig.ENABLE_UEFI_SECURE_BOOT.value(Boolean.class)); - } deviceBootOrderOperator.updateVmDeviceBootOrder(cmd, spec); cmd.setBootDev(toKvmBootDev(spec.getBootOrders())); @@ -5420,6 +5448,294 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private void handle(SecretHostGetMsg msg) { + SecretHostGetReply reply = new SecretHostGetReply(); + if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose()) || + msg.getKeyVersion() == null || StringUtils.isBlank(msg.getUsageInstance())) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "vmUuid, purpose, keyVersion and usageInstance are required for get secret")); + bus.reply(msg, reply); + return; + } + + String url = buildUrl(KVMConstant.KVM_GET_SECRET_PATH); + KVMAgentCommands.SecretHostGetCmd cmd = new KVMAgentCommands.SecretHostGetCmd(); + cmd.setVmUuid(msg.getVmUuid()); + cmd.setPurpose(msg.getPurpose()); + cmd.setKeyVersion(msg.getKeyVersion()); + cmd.setUsageInstance(msg.getUsageInstance()); + Map headers = new HashMap<>(); + headers.put(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID, getSelf().getUuid()); + Http http = new Http<>(url, cmd, KVMAgentCommands.SecretHostGetResponse.class); + http.runBeforeAsyncJsonPostExts(headers); + restf.asyncJsonPost(url, http.commandStr, headers, new JsonAsyncRESTCallback(msg, reply) { + @Override + public void fail(ErrorCode err) { + reply.setError(err != null ? err : operr(ORG_ZSTACK_KVM_10163, "get secret on agent failed")); + bus.reply(msg, reply); + } + + @Override + public void success(KVMAgentCommands.SecretHostGetResponse rsp) { + if (rsp != null && rsp.isSuccess() && StringUtils.isNotBlank(rsp.getSecretUuid())) { + reply.setSecretUuid(rsp.getSecretUuid()); + } else if (rsp != null && rsp.isSuccess()) { + ErrorCode err = new ErrorCode(); + err.setCode(SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND); + err.setDetails("get secret succeeded but secretUuid is empty"); + reply.setError(err); + } else { + reply.setError(buildSecretAgentError(rsp, "get secret failed")); + } + bus.reply(msg, reply); + } + + @Override + public Class getReturnClass() { + return KVMAgentCommands.SecretHostGetResponse.class; + } + }, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC); + } + + private void handle(ResolveVtpmLibvirtSecretOnHypervisorMsg msg) { + ResolveVtpmLibvirtSecretOnHypervisorReply reply = new ResolveVtpmLibvirtSecretOnHypervisorReply(); + if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getHostUuid())) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "vmUuid and hostUuid are required for vTPM resolve libvirt secret uuid")); + bus.reply(msg, reply); + return; + } + + KVMAgentCommands.ResolveVtpmLibvirtSecretCmd cmd = new KVMAgentCommands.ResolveVtpmLibvirtSecretCmd(); + cmd.setVmUuid(msg.getVmUuid()); + KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); + kmsg.setCommand(cmd); + kmsg.setPath(KVMConstant.KVM_VTPM_RESOLVE_LIBVIRT_SECRET_UUID_PATH); + kmsg.setHostUuid(msg.getHostUuid()); + bus.makeTargetServiceIdByResourceUuid(kmsg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(kmsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + reply.setError(r.getError()); + bus.reply(msg, reply); + return; + } + KVMHostAsyncHttpCallReply kreply = r.castReply(); + KVMAgentCommands.ResolveVtpmLibvirtSecretResponse rsp = + kreply.toResponse(KVMAgentCommands.ResolveVtpmLibvirtSecretResponse.class); + if (rsp != null && rsp.isSuccess() && StringUtils.isNotBlank(rsp.getSecretUuid())) { + reply.setSecretUuid(rsp.getSecretUuid()); + } else if (rsp != null && rsp.isSuccess()) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "vTPM resolve succeeded but secretUuid is empty")); + } else { + reply.setError(buildSecretAgentError(rsp, "vTPM resolve libvirt secret uuid failed")); + } + bus.reply(msg, reply); + } + }); + } + + private void handle(SecretHostDefineMsg msg) { + SecretHostDefineReply reply = new SecretHostDefineReply(); + if (org.apache.commons.lang.StringUtils.isBlank(msg.getDekBase64())) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "dekBase64 is required")); + bus.reply(msg, reply); + return; + } + if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose()) || + msg.getKeyVersion() == null || StringUtils.isBlank(msg.getUsageInstance())) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "vmUuid, purpose, keyVersion and usageInstance are required for ensure secret")); + bus.reply(msg, reply); + return; + } + String hostUuid = getSelf().getUuid(); + HostKeyIdentityVO identity = HostKeyIdentityHelper.getHostKeyIdentity(dbf, hostUuid); + String pubKey = identity != null ? org.apache.commons.lang.StringUtils.trimToNull(identity.getPublicKey()) : null; + Boolean verifyOk = identity != null ? identity.getVerified() : null; + if (pubKey == null) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "no public key for host, connect/reconnect did not sync key")); + bus.reply(msg, reply); + return; + } + String storedFingerprint = StringUtils.trimToNull(identity.getFingerprint()); + String computed = HostKeyIdentityHelper.fingerprintFromPublicKey(pubKey); + if (storedFingerprint == null || !StringUtils.equals(storedFingerprint, computed)) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "host public key fingerprint mismatch, key may be corrupted or tampered")); + bus.reply(msg, reply); + return; + } + if (!Boolean.TRUE.equals(verifyOk)) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "host secret key verify not ok, not synced")); + bus.reply(msg, reply); + return; + } + byte[] dekRaw; + try { + dekRaw = java.util.Base64.getDecoder().decode(msg.getDekBase64().trim()); + } catch (IllegalArgumentException e) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "invalid dekBase64: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + if (dekRaw == null || dekRaw.length == 0) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "dekBase64 decoded to empty")); + bus.reply(msg, reply); + return; + } + + if (dekRaw.length > KVMConstant.MAX_DEK_BYTES) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "dekBase64 decoded payload is too large")); + bus.reply(msg, reply); + return; + } + + byte[] pubKeyBytes; + try { + pubKeyBytes = java.util.Base64.getDecoder().decode(pubKey); + } catch (IllegalArgumentException e) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "invalid host public key in DB: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + if (pubKeyBytes == null || pubKeyBytes.length != 32) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "host public key must be 32 bytes (X25519)")); + bus.reply(msg, reply); + return; + } + java.util.List sealers = pluginRegistry.getExtensionList(HostSecretEnvelopeCryptoExtensionPoint.class); + if (sealers == null || sealers.isEmpty()) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "host secret envelope sealer not available (premium crypto module required)")); + bus.reply(msg, reply); + return; + } + byte[] envelope; + try { + envelope = sealers.get(0).seal(pubKeyBytes, dekRaw); + } catch (Exception e) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "HPKE seal failed: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + String envelopeDekBase64 = java.util.Base64.getEncoder().encodeToString(envelope); + String url = buildUrl(KVMConstant.KVM_ENSURE_SECRET_PATH); + KVMAgentCommands.SecretHostDefineCmd cmd = new KVMAgentCommands.SecretHostDefineCmd(); + cmd.setEncryptedDek(envelopeDekBase64); + cmd.setVmUuid(msg.getVmUuid()); + cmd.setPurpose(msg.getPurpose()); + cmd.setKeyVersion(msg.getKeyVersion()); + cmd.setSecretUuid(msg.getSecretUuid()); + cmd.setDescription(msg.getDescription() != null ? msg.getDescription() : ""); + cmd.setUsageInstance(msg.getUsageInstance()); + Map headers = new HashMap<>(); + headers.put(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID, getSelf().getUuid()); + Http http = new Http<>(url, cmd, KVMAgentCommands.SecretHostDefineResponse.class); + http.runBeforeAsyncJsonPostExts(headers); + restf.asyncJsonPost(url, http.commandStr, headers, new JsonAsyncRESTCallback(msg, reply) { + @Override + public void fail(ErrorCode err) { + reply.setError(err != null ? err : operr(ORG_ZSTACK_KVM_10163, "ensure secret on agent failed")); + bus.reply(msg, reply); + } + + @Override + public void success(KVMAgentCommands.SecretHostDefineResponse rsp) { + if (rsp != null && rsp.isSuccess()) { + if (rsp.getSecretUuid() != null) { + reply.setSecretUuid(rsp.getSecretUuid()); + } + } else { + reply.setError(buildSecretAgentError(rsp, "ensure secret failed")); + } + bus.reply(msg, reply); + } + + @Override + public Class getReturnClass() { + return KVMAgentCommands.SecretHostDefineResponse.class; + } + }, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC); + } + + private void handle(SecretHostDeleteMsg msg) { + SecretHostDeleteReply reply = new SecretHostDeleteReply(); + if (StringUtils.isBlank(msg.getVmUuid()) || StringUtils.isBlank(msg.getPurpose()) || + StringUtils.isBlank(msg.getUsageInstance()) || msg.getKeyVersion() == null) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "vmUuid, purpose, keyVersion and usageInstance are required for delete secret")); + bus.reply(msg, reply); + return; + } + + String url = buildUrl(KVMConstant.KVM_DELETE_SECRET_PATH); + KVMAgentCommands.SecretHostDeleteCmd cmd = new KVMAgentCommands.SecretHostDeleteCmd(); + cmd.setVmUuid(msg.getVmUuid()); + cmd.setPurpose(msg.getPurpose()); + cmd.setKeyVersion(msg.getKeyVersion()); + cmd.setUsageInstance(msg.getUsageInstance()); + Map headers = new HashMap<>(); + headers.put(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID, getSelf().getUuid()); + Http http = new Http<>(url, cmd, KVMAgentCommands.SecretHostDeleteResponse.class); + http.runBeforeAsyncJsonPostExts(headers); + restf.asyncJsonPost(url, http.commandStr, headers, new JsonAsyncRESTCallback(msg, reply) { + @Override + public void fail(ErrorCode err) { + reply.setError(err != null ? err : operr(ORG_ZSTACK_KVM_10163, "delete secret on agent failed")); + bus.reply(msg, reply); + } + + @Override + public void success(KVMAgentCommands.SecretHostDeleteResponse rsp) { + if (rsp == null || !rsp.isSuccess()) { + ErrorCode agentErr = buildSecretAgentError(rsp, "delete secret failed"); + if (!SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND.equals(agentErr.getCode())) { + reply.setError(agentErr); + } + } + bus.reply(msg, reply); + } + + @Override + public Class getReturnClass() { + return KVMAgentCommands.SecretHostDeleteResponse.class; + } + }, TimeUnit.SECONDS, KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC); + } + + private ErrorCode buildSecretAgentError(KVMAgentCommands.AgentResponse rsp, String defaultMessage) { + String raw = rsp != null ? StringUtils.trimToNull(rsp.getError()) : null; + if (raw == null) { + return operr(ORG_ZSTACK_KVM_10163, defaultMessage); + } + String stable = stableKeyAgentErrorCodeFromRawMessage(raw); + if (stable != null) { + ErrorCode err = new ErrorCode(); + err.setCode(stable); + err.setDetails(raw); + return err; + } + return operr(ORG_ZSTACK_KVM_10163, "%s: %s", defaultMessage, raw); + } + + private static String stableKeyAgentErrorCodeFromRawMessage(String raw) { + String t = raw.trim(); + int colon = t.indexOf(':'); + String head = (colon >= 0 ? t.substring(0, colon) : t).trim(); + if (head.isEmpty()) { + return null; + } + if (SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND.equals(head)) { + return head; + } + if (head.startsWith("KEY_AGENT_")) { + return head; + } + return null; + } + @Override protected void deleteTakeOverFlag(Completion completion) { if (CoreGlobalProperty.UNIT_TEST_ON) { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostFactory.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostFactory.java index 7179271b967..c0bf3bac827 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostFactory.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHostFactory.java @@ -753,6 +753,14 @@ public String handleSyncHttpCall(TransmitVmOperationToMnCmd cmd) { return null; }); + restf.registerSyncHttpCallHandler(KVMConstant.KVM_REPORT_VM_HOST_FILE_CHANGED, ReportVmHostFileChangedCmd.class, cmd -> { + VmCanonicalEvents.VmHostFileChangedData data = new VmCanonicalEvents.VmHostFileChangedData(); + data.setHostUuid(cmd.hostUuid); + data.setVmUuid(cmd.vmUuid); + data.setTypes(cmd.types); + evf.fire(VmCanonicalEvents.VM_HOST_FILE_CHANGED_PATH, data); + return null; + }); restf.registerSyncHttpCallHandler(KVMConstant.KVM_HOST_PHYSICAL_NIC_ALARM_EVENT, KVMAgentCommands.PhysicalNicAlarmEventCmd.class, cmd -> { if (!isNeedAlarmNic(cmd.host, cmd.nic, cmd.bond)) { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMSystemTags.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMSystemTags.java index b0e4cc3fdeb..f8f81ec90f8 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMSystemTags.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMSystemTags.java @@ -6,6 +6,7 @@ import org.zstack.header.tag.TagDefinition; import org.zstack.header.vm.VmInstanceVO; import org.zstack.header.vm.VmNicVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO; import org.zstack.header.volume.VolumeVO; import org.zstack.kvm.xmlhook.XmlHookVO; import org.zstack.tag.PatternedSystemTag; @@ -72,4 +73,16 @@ public class KVMSystemTags { public static final String LIBVIRT_CAPABILITIES_TOKEN = "libvirtCapabilities"; public static PatternedSystemTag LIBVIRT_CAPABILITIES = new PatternedSystemTag(String.format("libvirtCapabilities::{%s}", LIBVIRT_CAPABILITIES_TOKEN), HostVO.class); + + public static final String EDK_RPM_TOKEN = "edkRpm"; + public static PatternedSystemTag VM_EDK = + new PatternedSystemTag(String.format("vm::edk::{%s}", EDK_RPM_TOKEN), VmInstanceVO.class); + + public static final String SWTPM_VERSION_TOKEN = "version"; + public static PatternedSystemTag SWTPM_VERSION = + new PatternedSystemTag(String.format("swtpm::{%s}", SWTPM_VERSION_TOKEN), HostVO.class); + + public static final String TPM_KEY_PROVIDER_NAME_TOKEN = "keyProviderName"; + public static PatternedSystemTag TPM_KEY_PROVIDER_NAME = + new PatternedSystemTag(String.format("tpm::keyProvider::{%s}", TPM_KEY_PROVIDER_NAME_TOKEN), VmHostBackupFileVO.class); } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmCommandSender.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmCommandSender.java index 139f417c221..d8e711d6718 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmCommandSender.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmCommandSender.java @@ -69,6 +69,11 @@ public KvmCommandSender(String hostUuid, boolean noStatusCheck) { DebugUtils.Assert(hostUuid != null, "hostUuid cannot be null"); } + public KvmCommandSender disableHostStatusCheck() { + this.noStatusCheck = true; + return this; + } + public void send(final Object cmd, final String path, final KvmCommandFailureChecker checker, final SteppingSendCallback completion) { List msgs = hostUuids.stream().map(huuid -> { KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KvmResourceConfigExtension.java b/plugin/kvm/src/main/java/org/zstack/kvm/KvmResourceConfigExtension.java index 917648f9209..c116cfaf42c 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KvmResourceConfigExtension.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KvmResourceConfigExtension.java @@ -1,6 +1,7 @@ package org.zstack.kvm; import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.VmGlobalConfig; import org.zstack.header.vm.ArchiveResourceConfigBundle; import org.zstack.header.vm.ResourceConfigMemorySnapshotExtensionPoint; import org.zstack.resourceconfig.ResourceConfigFacade; @@ -21,6 +22,13 @@ public List getNeedToArchiveRe bundle.setValue(rcf.getResourceConfigValue(KVMGlobalConfig.NESTED_VIRTUALIZATION, resourceUuid, String.class)); bundleList.add(bundle); + ArchiveResourceConfigBundle.ResourceConfigBundle secureBootBundle = + new ArchiveResourceConfigBundle.ResourceConfigBundle(); + secureBootBundle.setResourceUuid(resourceUuid); + secureBootBundle.setIdentity(VmGlobalConfig.ENABLE_UEFI_SECURE_BOOT.getIdentity()); + secureBootBundle.setValue(rcf.getResourceConfigValue(VmGlobalConfig.ENABLE_UEFI_SECURE_BOOT, resourceUuid, String.class)); + bundleList.add(secureBootBundle); + if (Boolean.TRUE.equals(KVMHostUtils.isWindowsVmByUuid(resourceUuid))) { Boolean cpuHardwareVirtualization = rcf.getResourceConfigValue(KVMGlobalConfig.VM_CPU_HARDWARE_VIRTUALIZATION, resourceUuid, Boolean.class); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java new file mode 100644 index 00000000000..51713c6c7c2 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java @@ -0,0 +1,872 @@ +package org.zstack.kvm.efi; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.VmGlobalConfig; +import org.zstack.compute.vm.devices.VmTpmManager; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.core.workflow.SimpleFlowChain; +import org.zstack.header.core.Completion; +import org.zstack.header.core.NoErrorCompletion; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.exception.CloudRuntimeException; +import org.zstack.header.message.MessageReply; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.vm.APICreateVmInstanceMsg.DiskAO; +import org.zstack.header.vm.PreVmInstantiateResourceExtensionPoint; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.AfterReimageVmInstanceExtensionPoint; +import org.zstack.header.vm.BeforeHaStartVmInstanceExtensionPoint; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceMigrateExtensionPoint; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.vm.VmInstantiateResourceException; +import org.zstack.header.vm.VmMigrationType; +import org.zstack.header.vm.VmPreMigrationExtensionPoint; +import org.zstack.header.vm.VmReleaseResourceExtensionPoint; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileContentVO; +import org.zstack.header.vm.additions.VmHostFileContentVO_; +import org.zstack.header.vm.additions.VmHostFileOperation; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.header.storage.snapshot.ConsistentType; +import org.zstack.header.storage.snapshot.CreateVolumesSnapshotOverlayInnerMsg; +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmReply; +import org.zstack.header.storage.snapshot.VolumeSnapshotCreationExtensionPoint; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupInventory; +import org.zstack.header.vm.devices.NvRamSpec; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMAgentCommands.*; +import org.zstack.kvm.KVMGlobalConfig; +import org.zstack.kvm.KVMHostInventory; +import org.zstack.kvm.KVMStartVmExtensionPoint; +import org.zstack.kvm.KvmCommandSender; +import org.zstack.kvm.KvmResponseWrapper; +import org.zstack.kvm.VolumeTO; +import org.zstack.kvm.vmfiles.message.SyncVmHostFilesFromHostMsg; +import org.zstack.resourceconfig.ResourceConfig; +import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Tuple; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.zstack.core.Platform.operr; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.PreMigration; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.PostMigration; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.BeforeHaStart; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.PrepareReRead; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.PrepareRead; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.ResourceRelease; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.SnapshotGroupOnlineBackup; +import static org.zstack.header.vm.additions.VmHostFileType.NvRam; +import static org.zstack.kvm.KVMConstant.*; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class KvmSecureBootExtensions implements KVMStartVmExtensionPoint, + PreVmInstantiateResourceExtensionPoint, + VmPreMigrationExtensionPoint, + AfterReimageVmInstanceExtensionPoint, + VmReleaseResourceExtensionPoint, + VmInstanceMigrateExtensionPoint, + VolumeSnapshotCreationExtensionPoint, + BeforeHaStartVmInstanceExtensionPoint { + private static final CLogger logger = Utils.getLogger(KvmSecureBootExtensions.class); + + @Autowired + private CloudBus bus; + @Autowired + private ResourceConfigFacade resourceConfigFacade; + @Autowired + private DatabaseFacade databaseFacade; + @Autowired + private KvmVmHostFileFactory vmHostFileFactory; + @Autowired + private KvmSecureBootManager secureBootManager; + + private final Object hostFileLock = new Object(); + + @Override + public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { + if (isUefiBootMode(cmd.getBootMode())) { + ResourceConfig resourceConfig; + resourceConfig = resourceConfigFacade.getResourceConfig(VmGlobalConfig.ENABLE_UEFI_SECURE_BOOT.getIdentity()); + cmd.setSecureBoot(resourceConfig.getResourceConfigValue(spec.getVmInventory().getUuid(), Boolean.class)); + + resourceConfig = resourceConfigFacade.getResourceConfig(KVMGlobalConfig.VM_EDK_VERSION_CONFIG.getIdentity()); + final String edkVersion = resourceConfig.getResourceConfigValue(spec.getVmInventory().getUuid(), String.class); + if (!Objects.equals(edkVersion, EDK_VERSION_NONE)) { + cmd.setEdkVersion(edkVersion); + } + } + + final NvRamSpec nvRam = spec.getDevicesSpec().getNvRam(); + if (nvRam != null && nvRam.isNeedRegister()) { + prepareNvRamToStartVmCmd(cmd, nvRam, host); + } + } + + private void prepareNvRamToStartVmCmd(KVMAgentCommands.StartVmCmd cmd, NvRamSpec nvRam, KVMHostInventory host) { + VolumeTO volume = new VolumeTO(); + volume.setDeviceType(VolumeTO.FILE); + volume.setInstallPath(buildNvramFilePath(cmd.getVmInstanceUuid())); + volume.setVolumeUuid(null); // not a volume + cmd.setNvRam(volume); + + synchronized (hostFileLock) { + final Timestamp now = Timestamp.from(Instant.now()); + VmHostFileVO nvRamFile = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, cmd.getVmInstanceUuid()) + .eq(VmHostFileVO_.type, NvRam) + .eq(VmHostFileVO_.hostUuid, host.getUuid()) + .find(); + if (nvRamFile == null) { + nvRamFile = new VmHostFileVO(); + nvRamFile.setUuid(Platform.getUuid()); + nvRamFile.setHostUuid(host.getUuid()); + nvRamFile.setVmInstanceUuid(cmd.getVmInstanceUuid()); + nvRamFile.setType(NvRam); + nvRamFile.setPath(volume.getInstallPath()); + nvRamFile.setCreateDate(now); + nvRamFile.setResourceName("NvRam file for " + cmd.getVmInstanceUuid()); + databaseFacade.persist(nvRamFile); + } else { + SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.uuid, nvRamFile.getUuid()) + .set(VmHostFileVO_.path, volume.getInstallPath()) + .set(VmHostFileVO_.lastOpDate, now) + .update(); + } + } + } + + @Override + public void preMigrateVm(VmInstanceInventory inv, String destHostUuid, Completion completion) { + prepareNvRamBeforeMigration(inv, destHostUuid, completion); + } + + @Override + public void preVmMigration(VmInstanceInventory vm, VmMigrationType type, String dstHostUuid, Completion completion) { + completion.success(); // use preMigrateVm instead of preVmMigration to prevent from handle twice + } + + private void prepareNvRamBeforeMigration(VmInstanceInventory vm, String dstHostUuid, Completion completion) { + if (dstHostUuid == null) { + completion.success(); + return; + } + + boolean needRegisterNvRam = vmHostFileFactory.needRegister(NvRam, vm.getUuid()); + if (!needRegisterNvRam) { + completion.success(); + return; + } + + SimpleFlowChain.of("prepare-nvram-before-vm-" + vm.getUuid() + "-migrate") + .then("sync-vm-host-files-before-migration", trigger -> { + String vmUuid = vm.getUuid(); + + Tuple tuple = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.hostUuid, VmInstanceVO_.lastHostUuid) + .findTuple(); + String hostUuid = tuple.get(0, String.class); + if (hostUuid == null) { + hostUuid = tuple.get(1, String.class); + } + + List fileList = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .list(); + + if (fileList.isEmpty()) { + trigger.next(); + return; + } + + if (fileList.stream().allMatch(it -> it.getChangeDate() == null)) { + logger.debug(String.format("skip syncing VM host files before migration for VM[uuid=%s], " + + "All files are clean (changeDate is null)", vmUuid)); + trigger.next(); + return; + } + + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(hostUuid); + syncMsg.setVmUuid(vmUuid); + syncMsg.setSyncReason(PreMigration.reason()); + + for (VmHostFileVO file : fileList) { + if (file.getType() == NvRam) { + syncMsg.setNvRamPath(file.getPath()); + } else if (file.getType() == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(file.getPath()); + } + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + trigger.next(); + return; + } + trigger.fail(reply.getError()); + } + }); + }) + .then("prepare-nvram-folder-on-dest-host", trigger -> { + VmHostFileTO to = new VmHostFileTO(); + to.setPath(buildNvramFilePath(vm.getUuid())); + to.setType(NvRam.toString()); + to.setOperation(VmHostFileOperation.Prepare.toString()); + + RewriteVmHostFilesContext context = new RewriteVmHostFilesContext(); + context.hostUuid = dstHostUuid; + context.hostFiles = list(to); + + rewriteVmHostFiles(context, new Completion(trigger) { + @Override + public void success() { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + }) + // TpmState folder is not needed to prepare + .propagateExceptionTo(completion) + .done(completion::success) + .error(completion::fail) + .start(); + } + + public static class RewriteVmHostFilesContext { + public String hostUuid; + public List hostFiles; + } + + public void rewriteVmHostFiles(RewriteVmHostFilesContext context, Completion completion) { + KvmCommandSender sender = new KvmCommandSender(context.hostUuid); + KVMAgentCommands.WriteVmHostFileContentCmd cmd = new KVMAgentCommands.WriteVmHostFileContentCmd(); + cmd.setHostFiles(context.hostFiles); + + sender.send(cmd, WRITE_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.WriteVmHostFileContentResponse writeRsp = wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class); + return writeRsp.isSuccess() ? null : + operr(ORG_ZSTACK_KVM_10163, "failed to write file content response: %s", writeRsp.getError()); + }, new ReturnValueCompletion(completion) { + @Override + public void success(KvmResponseWrapper wrapper) { + KVMAgentCommands.WriteVmHostFileContentResponse writeRsp = wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class); + if (!writeRsp.isSuccess()) { + completion.fail(operr(ORG_ZSTACK_KVM_10163, + "failed to write file content response: %s", writeRsp.getError())); + return; + } + completion.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void startVmOnKvmSuccess(KVMHostInventory host, VmInstanceSpec spec) { + // do-nothing + } + + @Override + public void startVmOnKvmFailed(KVMHostInventory host, VmInstanceSpec spec, ErrorCode err) { + // do-nothing + } + + private boolean isUefiBootMode(String bootMode) { + return VmTpmManager.isUefiBootMode(bootMode); + } + + @Override + public void preBeforeInstantiateVmResource(VmInstanceSpec spec) throws VmInstantiateResourceException { + // do-nothing + } + + @Override + public void preInstantiateVmResource(VmInstanceSpec spec, Completion completion) { + final NvRamSpec nvRamSpec = spec.getDevicesSpec() == null ? null : spec.getDevicesSpec().getNvRam(); + if (nvRamSpec == null) { + completion.success(); + return; + } + + PrepareHostFileContext context = new PrepareHostFileContext(); + context.hostUuid = spec.getDestHost().getUuid(); + context.vmUuid = spec.getVmInventory().getUuid(); + context.type = NvRam; + context.backupUuid = nvRamSpec.getBackupFileUuid(); + context.syncReason = "pre-instantiate VM resource"; + prepareHostFileOnHost(context, completion); + } + + public static class PrepareHostFileContext { + public String hostUuid; + public String vmUuid; + public VmHostFileType type; + public String backupUuid; + public String syncReason; + + public String path; + // whether the NvRam is on the same host as before + private boolean sameHost = false; + private boolean syncFromOriginHostSuccess = false; + private boolean writeSuccess = false; + private VmHostFileVO vmHostFile; + private VmHostBackupFileVO vmBackupFileVO; + + // property: VmHostBackupFileVO (when "backupUuid" is set) + // > VmHostFileVO (read success) + // > VmHostFileVO (read fail, but has content) + // > VmHostBackupFileVO (vmInstanceUuid matched) <- read this only if VmHostFileVO is not exist + } + + @SuppressWarnings("rawtypes") + public void prepareHostFileOnHost(PrepareHostFileContext context, Completion completion) { + SimpleFlowChain chain = new SimpleFlowChain(); + chain.setName("prepare-vm-host-file"); + chain.then(new NoRollbackFlow() { + String __name__ = "read-vm-host-file-from-origin-host"; + + @Override + public boolean skip(Map data) { + return StringUtils.isNotBlank(context.backupUuid); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + VmHostFileVO vmHostFile = context.vmHostFile = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.type, context.type) + .eq(VmHostFileVO_.vmInstanceUuid, context.vmUuid) + .orderByDesc(VmHostFileVO_.lastOpDate) + .limit(1) + .find(); + if (vmHostFile == null) { + logger.debug(String.format("skip to read/write %s host file for VM[vmUuid=%s]: file is not registered in MN", + context.type, context.vmUuid)); + trigger.next(); + return; + } + + context.sameHost = vmHostFile.getHostUuid().equals(context.hostUuid); + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(vmHostFile.getHostUuid()); + syncMsg.setVmUuid(context.vmUuid); + syncMsg.setSyncReason(PrepareRead.reason(context.syncReason)); + + if (vmHostFile.getType() == NvRam) { + context.path = vmHostFile.getPath(); + syncMsg.setNvRamPath(context.path); + } else if (vmHostFile.getType() == VmHostFileType.TpmState) { + context.path = vmHostFile.getPath(); + syncMsg.setTpmStateFolder(context.path); + } else { + throw new CloudRuntimeException("unsupported vm host file type: " + vmHostFile.getType()); + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + context.syncFromOriginHostSuccess = true; + trigger.next(); + return; + } + + logger.warn(String.format("failed to read vm host file for VM[vmUuid=%s] but still continue: %s", + context.vmUuid, reply.getError().getReadableDetails())); + trigger.next(); + } + }); + } + }).then(new NoRollbackFlow() { + String __name__ = "check-content-for-vm-host-file-if-sync-failed"; + + @Override + public boolean skip(Map data) { + return context.syncFromOriginHostSuccess || context.vmHostFile == null; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + Timestamp latestChangeDate = Q.New(VmHostFileVO.class) + .select(VmHostFileVO_.changeDate) + .eq(VmHostFileVO_.uuid, context.vmHostFile.getUuid()) + .findValue(); + if (latestChangeDate != null && !context.sameHost) { + trigger.fail(operr( + "vm host file content on management node may be stale: change is pending on host " + + "but sync from origin host failed, cannot use cached content[vmUuid=%s, type=%s]", + context.vmUuid, context.type)); + return; + } + + if (!Q.New(VmHostFileContentVO.class) + .eq(VmHostFileContentVO_.uuid, context.vmHostFile.getUuid()) + .isExists()) { + logger.debug(String.format( + "VmHostFileVO[vmUuid=%s, hostUuid=%s, type=%s] has no content", + context.vmUuid, context.hostUuid, context.type)); + context.vmHostFile = null; + } + trigger.next(); + } + }).then(new NoRollbackFlow() { + String __name__ = "read-vm-host-file-from-backup"; + + @Override + public boolean skip(Map data) { + return context.vmHostFile != null; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + if (context.backupUuid == null) { + context.vmBackupFileVO = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.type, context.type) + .eq(VmHostBackupFileVO_.resourceUuid, context.vmUuid) + .orderByDesc(VmHostBackupFileVO_.lastOpDate) + .limit(1) + .find(); + } else { + context.vmBackupFileVO = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.uuid, context.backupUuid) + .find(); + if (context.vmBackupFileVO == null) { + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "cannot find matched vm-host backup file[backupUuid:%s]", + context.backupUuid)); + return; + } + } + if (context.vmBackupFileVO != null) { + logger.debug(String.format("use %s[type=%s] VM-host backup file for VM[uuid=%s]", + context.vmBackupFileVO.getUuid(), context.type, context.vmUuid)); + context.path = buildPathForVmHostFileType(context.type, context.vmUuid); + } + trigger.next(); + } + }).then(new NoRollbackFlow() { + String __name__ = "write-vm-host-file-to-dest-host"; + + @Override + public boolean skip(Map data) { + return (context.vmHostFile == null && context.vmBackupFileVO == null) + || (context.sameHost && context.vmHostFile != null); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + String contentUuid = context.vmHostFile == null ? + context.vmBackupFileVO.getUuid() : context.vmHostFile.getUuid(); + VmHostFileContentVO content = Q.New(VmHostFileContentVO.class) + .eq(VmHostFileContentVO_.uuid, contentUuid) + .find(); + if (content == null) { + logger.debug(String.format("skip to write vm host file for VM[vmUuid=%s]: file content is not saved in MN", + context.vmUuid)); + trigger.next(); + return; + } + + VmHostFileTO to = new VmHostFileTO(); + to.setPath(context.path); + to.setType(context.type.toString()); + to.setFileFormat(content.getFormat().toString()); + to.setOperation(VmHostFileOperation.Write.toString()); + + String contentBase64 = Base64.getEncoder().encodeToString(content.getContent()); + to.setContentBase64(contentBase64); + + RewriteVmHostFilesContext rewriteContext = new RewriteVmHostFilesContext(); + rewriteContext.hostUuid = context.hostUuid; + rewriteContext.hostFiles = Collections.singletonList(to); + + rewriteVmHostFiles(rewriteContext, new Completion(trigger) { + @Override + public void success() { + context.writeSuccess = true; + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + }).then(new NoRollbackFlow() { + String __name__ = "re-read-vm-host-file-from-dest-host"; + + @Override + public boolean skip(Map data) { + return !context.writeSuccess; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(context.hostUuid); + syncMsg.setVmUuid(context.vmUuid); + syncMsg.setSyncReason(PrepareReRead.reason(context.syncReason)); + + if (context.type == NvRam) { + syncMsg.setNvRamPath(context.path); + } else if (context.type == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(context.path); + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + } + }).done(new FlowDoneHandler(completion) { + @Override + public void handle(Map data) { + completion.success(); + } + }).error(new FlowErrorHandler(completion) { + @Override + public void handle(ErrorCode errCode, Map data) { + completion.fail(errCode); + } + }).start(); + } + + @Override + public void preReleaseVmResource(VmInstanceSpec spec, Completion completion) { + completion.success(); + } + + static class NvRamVolumeContext { + String vmUuid; + DiskAO nvRamSpec; + VmInstanceSpec spec; + + VolumeInventory inventory; + } + + @Override + public void afterReimageVmInstance(VolumeInventory inventory) { + String vmUuid = inventory.getVmInstanceUuid(); + if (vmUuid == null) { + return; + } + + SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .delete(); + secureBootManager.cleanVmHostBackupFile(vmUuid); + + logger.debug(String.format("reset TPM state for VM[uuid:%s] after reimage: " + + "deleted all VmHostFileVO and VmHostBackupFileVO records", vmUuid)); + } + + @Override + public void releaseVmResource(VmInstanceSpec spec, Completion completion) { + if (spec.getDestHost() == null) { + completion.success(); + return; + } + + String hostUuid = spec.getDestHost().getUuid(); + String vmUuid = spec.getVmInventory().getUuid(); + List vmHostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .list(); + if (vmHostFiles.isEmpty()) { + completion.success(); + return; + } + + logger.info(String.format("try to sync VM host file[vmUuid=%s] from host[uuid=%s]", vmUuid, hostUuid)); + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(hostUuid); + syncMsg.setVmUuid(vmUuid); + syncMsg.setSyncReason(ResourceRelease.reason()); + + for (VmHostFileVO file : vmHostFiles) { + if (file.getType() == NvRam) { + syncMsg.setNvRamPath(file.getPath()); + } else if (file.getType() == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(file.getPath()); + } else { + logger.warn(String.format("unsupported vm host file type: %s, skip syncing for VM[uuid:%s] from host[uuid:%s]", + file.getType(), vmUuid, hostUuid)); + } + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.warn(String.format("failed to sync VM host file[vmUuid=%s] from host[uuid=%s] but still continue: %s", + vmUuid, hostUuid, reply.getError().getReadableDetails())); + } + completion.success(); + } + }); + } + + @Override + public void beforeMigrateVm(VmInstanceInventory inv, String destHostUuid) { + } + + @Override + public void beforeHaStartVmInstance(String vmUuid, String judgerClassName, List softAvoidHostUuids, Completion completion) { + Tuple hostUuidTuple = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid, VmInstanceVO_.lastHostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findTuple(); + if (hostUuidTuple == null) { + completion.success(); + return; + } + + String hostUuid = hostUuidTuple.get(0, String.class); + final String finalHostUuid = (hostUuid == null) ? hostUuidTuple.get(1, String.class) : hostUuid; + if (finalHostUuid == null) { + logger.debug("skip HA file sync: VM has no host/lastHost record"); + completion.success(); + return; + } + + List vmHostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, finalHostUuid) + .list(); + if (vmHostFiles.isEmpty()) { + completion.success(); + return; + } + + logger.info(String.format("try to sync VM host file[vmUuid=%s] from last host[uuid=%s] before HA start", + vmUuid, finalHostUuid)); + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(finalHostUuid); + syncMsg.setVmUuid(vmUuid); + syncMsg.setSyncReason(BeforeHaStart.reason()); + + for (VmHostFileVO file : vmHostFiles) { + if (file.getType() == NvRam) { + syncMsg.setNvRamPath(file.getPath()); + } else if (file.getType() == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(file.getPath()); + } else { + logger.warn(String.format( + "unsupported vm host file type: %s, skip syncing for VM[uuid:%s] from host[uuid:%s]", + file.getType(), vmUuid, finalHostUuid)); + } + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.warn(String.format("failed to sync VM host file[vmUuid=%s] from last host[uuid=%s] before HA start, but tolerated: %s", + vmUuid, finalHostUuid, reply.getError().getReadableDetails())); + } + completion.success(); + } + }); + } + + @Override + public void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid, NoErrorCompletion completion) { + String destHostUuid = inv.getHostUuid(); + String vmUuid = inv.getUuid(); + + List vmHostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, srcHostUuid) + .list(); + if (vmHostFiles.isEmpty()) { + completion.done(); + return; + } + + // clean up stale VmHostFileVO/VmHostFileContentVO on dest host + // before sync creates new records + List staleFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, destHostUuid) + .list(); + if (!staleFiles.isEmpty()) { + List staleUuids = staleFiles.stream() + .map(VmHostFileVO::getUuid) + .collect(java.util.stream.Collectors.toList()); + SQL.New(VmHostFileContentVO.class) + .in(VmHostFileContentVO_.uuid, staleUuids) + .delete(); + SQL.New(VmHostFileVO.class) + .in(VmHostFileVO_.uuid, staleUuids) + .delete(); + logger.debug(String.format("cleaned up %d stale VmHostFileVO/Content on dest host[uuid=%s] for VM[uuid=%s]", + staleFiles.size(), destHostUuid, vmUuid)); + } + + logger.info(String.format("sync VM host file[vmUuid=%s] from dest host[uuid=%s] after migration", + vmUuid, destHostUuid)); + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(destHostUuid); + syncMsg.setVmUuid(vmUuid); + syncMsg.setSyncReason(PostMigration.reason()); + + for (VmHostFileVO file : vmHostFiles) { + if (file.getType() == NvRam) { + syncMsg.setNvRamPath(file.getPath()); + } else if (file.getType() == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(file.getPath()); + } else { + logger.warn(String.format("unsupported vm host file type: %s, skip syncing for VM[uuid:%s]", + file.getType(), vmUuid)); + } + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.warn(String.format("failed to sync VM host file[vmUuid=%s] from host[uuid=%s] after migration, but tolerated: %s", + vmUuid, destHostUuid, reply.getError().getReadableDetails())); + } + completion.done(); + } + }); + } + + @Override + public void afterVolumeLiveSnapshotGroupCreatedOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmReply treply, + Completion completion) { + if (treply == null || !treply.isSuccess()) { + completion.success(); + return; + } + + if (!msg.isBackupHostFileIfNeeded()) { + completion.success(); + return; + } + + String vmUuid = msg.getLockedVmInstanceUuids().get(0); + String hostUuid = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findValue(); + + List hostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .list(); + + if (hostFiles.isEmpty()) { + completion.success(); + return; + } + + String tempResourceUuid = Platform.getUuid(); + + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setVmUuid(vmUuid); + syncMsg.setHostUuid(hostUuid); + + for (VmHostFileVO file : hostFiles) { + if (file.getType() == NvRam) { + syncMsg.setNvRamPath(buildNvramSnapshotBackupFilePath(vmUuid)); + } else if (file.getType() == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(buildTpmStateSnapshotBackupFilePath(vmUuid)); + } + } + + syncMsg.setSyncReason(SnapshotGroupOnlineBackup.reason()); + syncMsg.setSyncToBackup(true); + syncMsg.setBackupResourceUuid(tempResourceUuid); + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + treply.setHostBackupTempResourceUuid(tempResourceUuid); + logger.debug(String.format("synced backup host files for vm[uuid:%s] to VmHostBackupFileVO[resourceUuid:%s]", + vmUuid, tempResourceUuid)); + } else { + logger.warn(String.format("failed to sync backup host files for vm[uuid:%s] during online snapshot, " + + "but tolerated: %s", vmUuid, reply.getError().getReadableDetails())); + } + completion.success(); + } + }); + } + + @Override + public void afterVolumeLiveSnapshotGroupCreationFailsOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmReply treply) { + // No cleanup needed — backup files on agent side are ephemeral + } + + @Override + public void afterVolumeSnapshotGroupCreated(VolumeSnapshotGroupInventory snapshotGroup, + ConsistentType consistentType, + Completion completion) { + completion.success(); + } + + @Override + public void afterVolumeSnapshotCreated(VolumeSnapshotInventory snapshot, Completion completion) { + completion.success(); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java new file mode 100644 index 00000000000..cd65bf8de3e --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java @@ -0,0 +1,1153 @@ +package org.zstack.kvm.efi; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend; +import org.zstack.core.Platform; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.EventCallback; +import org.zstack.core.cloudbus.EventFacadeImpl; +import org.zstack.core.cloudbus.MessageSafe; +import org.zstack.core.cloudbus.ResourceDestinationMaker; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.core.db.SQLBatch; +import org.zstack.core.timeout.TimeHelper; +import org.zstack.core.workflow.SimpleFlowChain; +import org.zstack.header.AbstractService; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.exception.CloudRuntimeException; +import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; +import org.zstack.header.vm.VmCanonicalEvents; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.additions.RestoreVmHostFileMsg; +import org.zstack.header.vm.additions.RestoreVmHostFileReply; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionReply; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileContentFormat; +import org.zstack.header.vm.additions.VmHostFileContentVO; +import org.zstack.header.vm.additions.VmHostFileContentVO_; +import org.zstack.header.vm.additions.VmHostFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostFileDeletionReply; +import org.zstack.header.vm.additions.VmHostFileManager; +import org.zstack.header.vm.additions.VmHostFileOperation; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KvmCommandSender; +import org.zstack.kvm.KvmResponseWrapper; +import org.zstack.kvm.vmfiles.message.BackupVmHostFileMsg; +import org.zstack.kvm.vmfiles.message.BackupVmHostFileOnHypervisorMsg; +import org.zstack.kvm.vmfiles.message.BackupVmHostFileOnHypervisorReply; +import org.zstack.kvm.vmfiles.message.BackupVmHostFileReply; +import org.zstack.kvm.vmfiles.message.CloneVmHostFileMsg; +import org.zstack.kvm.vmfiles.message.CloneVmHostFileReply; +import org.zstack.kvm.vmfiles.message.SyncVmHostFilesFromHostMsg; +import org.zstack.kvm.vmfiles.message.SyncVmHostFilesFromHostReply; +import org.zstack.resourceconfig.ResourceConfig; +import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.utils.DebugUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Tuple; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; + +import static org.zstack.compute.vm.VmGlobalConfig.RESET_TPM_AFTER_VM_CLONE; +import static org.zstack.core.Platform.operr; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.PostClone; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.Restore; +import static org.zstack.header.vm.additions.VmHostFileSyncReason.VmShutdown; +import static org.zstack.kvm.KVMAgentCommands.*; +import static org.zstack.kvm.KVMConstant.*; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.CollectionUtils.*; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class KvmSecureBootManager extends AbstractService implements VmHostFileManager { + private static final CLogger logger = Utils.getLogger(KvmSecureBootManager.class); + + @Autowired + private CloudBus bus; + @Autowired + private DatabaseFacade databaseFacade; + @Autowired + private EventFacadeImpl eventFacade; + @Autowired + private ResourceConfigFacade resourceConfigFacade; + @Autowired + private KvmVmHostFileFactory vmHostFileFactory; + @Autowired + private TimeHelper timeHelper; + @Autowired + private ResourceDestinationMaker resourceDestinationMaker; + @Autowired + private TpmEncryptedResourceKeyBackend resourceKeyBackend; + + @Override + public boolean start() { + setupCanonicalEvents(); + return true; + } + + @Override + public boolean stop() { + return true; + } + + @SuppressWarnings("rawtypes") + private void setupCanonicalEvents() { + eventFacade.on(VmCanonicalEvents.VM_LIBVIRT_REPORT_START, new EventCallback() { + @Override + protected void run(Map tokens, Object data) { + String vmUuid = (String) data; + boolean managedByMe = resourceDestinationMaker.isManagedByUs(vmUuid); + if (!managedByMe) { + return; + } + markVmHostFilesChanged(vmUuid); + } + }); + + eventFacade.on(VmCanonicalEvents.VM_LIBVIRT_REPORT_SHUTDOWN, new EventCallback() { + @Override + protected void run(Map tokens, Object data) { + String vmUuid = (String) data; + boolean managedByMe = resourceDestinationMaker.isManagedByUs(vmUuid); + if (!managedByMe) { + return; + } + + Tuple tuple = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid, VmInstanceVO_.lastHostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findTuple(); + if (tuple == null) { + return; + } + + String hostUuid = (String) tuple.get(0); + if (hostUuid == null) { + hostUuid = (String) tuple.get(1); + } + markVmHostFilesChanged(vmUuid, hostUuid); + + List hostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .in(VmHostFileVO_.type, list(VmHostFileType.NvRam, VmHostFileType.TpmState)) + .list(); + if (hostFiles.isEmpty()) { + return; + } + + VmHostFileVO nvRamFile = findOneOrNull(hostFiles, it -> it.getType() == VmHostFileType.NvRam); + VmHostFileVO tpmStateFile = findOneOrNull(hostFiles, it -> it.getType() == VmHostFileType.TpmState); + if (nvRamFile == null && tpmStateFile == null) { + return; + } + + SyncVmHostFilesFromHostMsg innerMessage = new SyncVmHostFilesFromHostMsg(); + innerMessage.setHostUuid(hostUuid); + innerMessage.setVmUuid(vmUuid); + innerMessage.setNvRamPath(nvRamFile == null ? null : nvRamFile.getPath()); + innerMessage.setTpmStateFolder(tpmStateFile == null ? null : tpmStateFile.getPath()); + innerMessage.setSyncReason(VmShutdown.reason()); + bus.makeLocalServiceId(innerMessage, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(innerMessage, new CloudBusCallBack(null) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + logger.info(String.format("success to read file content from host[uuid=%s]", + innerMessage.getHostUuid())); + } else { + logger.warn(String.format("failed to read file content from host[uuid=%s]: %s", + innerMessage.getHostUuid(), reply.getError().getReadableDetails())); + } + } + }); + } + }); + } + + /** + * Preemptive judgment: when a VM with TPM (or enabled secure boot) starts or shuts down, + * the NvRam/TpmState data must have changed, so mark the corresponding + * VmHostFileVO.changeDate to current time. + */ + private void markVmHostFilesChanged(String vmUuid) { + Tuple tuple = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid, VmInstanceVO_.lastHostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findTuple(); + if (tuple == null) { + return; + } + + String hostUuid = (String) tuple.get(0); + if (hostUuid == null) { + hostUuid = (String) tuple.get(1); + } + if (hostUuid == null) { + return; + } + + markVmHostFilesChanged(vmUuid, hostUuid); + } + + private void markVmHostFilesChanged(String vmUuid, String hostUuid) { + if (hostUuid == null) { + return; + } + + final Set types = vmHostFileFactory.vmHostFileTypeNeedRegisterForVm(vmUuid); + if (types.isEmpty()) { + return; + } + + Timestamp now = new Timestamp(timeHelper.getCurrentTimeMillis()); + long updated = SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .in(VmHostFileVO_.type, types) + .set(VmHostFileVO_.changeDate, now) + .update(); + + if (updated > 0) { + logger.debug(String.format("preemptively marked VmHostFiles as changed for VM[uuid:%s] on host[uuid:%s], %d records updated", + vmUuid, hostUuid, updated)); + } + } + + @Override + public String getId() { + return bus.makeLocalServiceId(VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + } + + @Override + @MessageSafe + public void handleMessage(Message msg) { + if (msg instanceof SyncVmHostFilesFromHostMsg) { + handle((SyncVmHostFilesFromHostMsg) msg); + } else if (msg instanceof CloneVmHostFileMsg) { + handle((CloneVmHostFileMsg) msg); + } else if (msg instanceof BackupVmHostFileMsg) { + handle((BackupVmHostFileMsg) msg); + } else if (msg instanceof RestoreVmHostFileMsg) { + handle((RestoreVmHostFileMsg) msg); + } else if (msg instanceof BackupVmHostFileOnHypervisorMsg) { + handle((BackupVmHostFileOnHypervisorMsg) msg); + } else if (msg instanceof VmHostFileDeletionMsg) { + handle((VmHostFileDeletionMsg) msg); + } else if (msg instanceof VmHostBackupFileDeletionMsg) { + handle((VmHostBackupFileDeletionMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + static class CloneVmHostFileContext { + List typesNeedClone = new ArrayList<>(); + List files = new ArrayList<>(); + List backupFiles = new ArrayList<>(); + List syncContexts = new ArrayList<>(); + } + + private void handle(SyncVmHostFilesFromHostMsg msg) { + KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid()) + .disableHostStatusCheck(); + + KVMAgentCommands.ReadVmHostFileContentCmd cmd = new KVMAgentCommands.ReadVmHostFileContentCmd(); + cmd.setHostFiles(new ArrayList<>()); + if (msg.getTpmStateFolder() != null) { + KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); + to.setPath(msg.getTpmStateFolder()); + to.setType(VmHostFileType.TpmState.toString()); + cmd.getHostFiles().add(to); + } + if (msg.getNvRamPath() != null) { + KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); + to.setPath(msg.getNvRamPath()); + to.setType(VmHostFileType.NvRam.toString()); + cmd.getHostFiles().add(to); + } + long now = timeHelper.getCurrentTimeMillis(); + + SyncVmHostFilesFromHostReply reply = new SyncVmHostFilesFromHostReply(); + sender.send(cmd, READ_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.ReadVmHostFileContentResponse readRsp = wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class); + return readRsp.isSuccess() ? null : + operr(ORG_ZSTACK_KVM_10163, "failed to read file content response: %s", readRsp.getError()); + }, new ReturnValueCompletion(msg) { + @Override + public void success(KvmResponseWrapper wrapper) { + KVMAgentCommands.ReadVmHostFileContentResponse readRsp = wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class); + if (!readRsp.isSuccess()) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, + "failed to read file content response: %s", readRsp.getError())); + bus.reply(msg, reply); + return; + } + + ErrorCode error; + if (msg.isSyncToBackup()) { + error = syncToBackupFiles(msg, readRsp); + } else { + error = syncToHostFiles(msg, cmd, readRsp, now); + } + + if (error != null) { + reply.setError(error); + } + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private ErrorCode syncToHostFiles(SyncVmHostFilesFromHostMsg msg, + KVMAgentCommands.ReadVmHostFileContentCmd cmd, + KVMAgentCommands.ReadVmHostFileContentResponse readRsp, + long timeBeforeSync) { + final List existsFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, msg.getVmUuid()) + .eq(VmHostFileVO_.hostUuid, msg.getHostUuid()) + .in(VmHostFileVO_.path, cmd.getPaths()) + .list(); + final List existsContentUuid; + if (!existsFiles.isEmpty()) { + existsContentUuid = Q.New(VmHostFileContentVO.class) + .in(VmHostFileContentVO_.uuid, transform(existsFiles, VmHostFileVO::getUuid)) + .select(VmHostFileContentVO_.uuid) + .listValues(); + } else { + existsContentUuid = Collections.emptyList(); + } + + Timestamp syncTime = new Timestamp(timeBeforeSync); + List errors = new ArrayList<>(); + for (String path : cmd.getPaths()) { + KVMAgentCommands.VmHostFileTO to = findOneOrNull(readRsp.getHostFiles(), item -> item.getPath().equals(path)); + if (to == null) { + continue; + } + if (to.getError() != null) { + errors.add(operr(ORG_ZSTACK_KVM_10163, "failed to read file %s: %s", path, to.getError()) + .withOpaque("path", path)); + continue; + } + + VmHostFileType type = Objects.equals(path, msg.getNvRamPath()) ? + VmHostFileType.NvRam : VmHostFileType.TpmState; + + VmHostFileVO file = findOneOrNull(existsFiles, item -> item.getPath().equals(path)); + boolean fileExists = file != null; + + if (fileExists) { + String fileUuid = file.getUuid(); + new SQLBatch() { + @Override + protected void scripts() { + sql(VmHostFileVO.class) + .eq(VmHostFileVO_.uuid, fileUuid) + .set(VmHostFileVO_.lastSyncReason, msg.getSyncReason()) + .set(VmHostFileVO_.lastSyncDate, syncTime) + .set(VmHostFileVO_.lastOpDate, syncTime) + .update(); + sql(VmHostFileVO.class) + .eq(VmHostFileVO_.uuid, fileUuid) + .lt(VmHostFileVO_.changeDate, syncTime) + .set(VmHostFileVO_.changeDate, null) // CAS update + .update(); + } + }.execute(); + + } else { + file = new VmHostFileVO(); + file.setUuid(Platform.getUuid()); + file.setHostUuid(msg.getHostUuid()); + file.setVmInstanceUuid(msg.getVmUuid()); + file.setPath(path); + file.setType(type); + file.setLastSyncReason(msg.getSyncReason()); + file.setLastSyncDate(syncTime); + file.setChangeDate(null); + file.setLastOpDate(syncTime); + file.setCreateDate(syncTime); + file.setResourceName(String.format("%s file for %s", type, msg.getVmUuid())); + databaseFacade.persist(file); + } + + byte[] bytes = Base64.getDecoder().decode(to.getContentBase64()); + if (existsContentUuid.contains(file.getUuid())) { + SQL.New(VmHostFileContentVO.class) + .eq(VmHostFileContentVO_.uuid, file.getUuid()) + .set(VmHostFileContentVO_.content, bytes) + .set(VmHostFileContentVO_.format, VmHostFileContentFormat.valueOf(to.getFileFormat())) + .set(VmHostFileContentVO_.lastOpDate, syncTime) + .update(); + } else { + VmHostFileContentVO content = new VmHostFileContentVO(); + content.setUuid(file.getUuid()); + content.setContent(bytes); + content.setFormat(VmHostFileContentFormat.valueOf(to.getFileFormat())); + content.setCreateDate(syncTime); + content.setLastOpDate(syncTime); + databaseFacade.persist(content); + } + + if (logger.isTraceEnabled()) { + logger.trace(String.format("persist/update VmHostFileContentVO [uuid=%s]", file.getUuid())); + } + } + + if (errors.isEmpty()) { + return null; + } + + return operr(ORG_ZSTACK_KVM_10163, "failed to read file content from host[uuid=%s]", msg.getHostUuid()) + .causedBy(errors); + } + + private ErrorCode syncToBackupFiles(SyncVmHostFilesFromHostMsg msg, + KVMAgentCommands.ReadVmHostFileContentResponse readRsp) { + if (msg.getBackupResourceUuid() == null || msg.getBackupResourceUuid().isEmpty()) { + return operr(ORG_ZSTACK_KVM_10163, "backupResourceUuid is required when syncToBackup is true"); + } + + // Query the source VmHostFileVO records for afterBackup callback + final List sourceHostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, msg.getVmUuid()) + .eq(VmHostFileVO_.hostUuid, msg.getHostUuid()) + .list(); + + List backupFilesToPersist = new ArrayList<>(); + List contentsToPersist = new ArrayList<>(); + Map backupFromMap = new HashMap<>(); + + List errors = new ArrayList<>(); + Timestamp now = Timestamp.from(Instant.now()); + + for (KVMAgentCommands.VmHostFileTO to : readRsp.getHostFiles()) { + if (to == null) { + continue; + } + if (to.getError() != null) { + errors.add(operr(ORG_ZSTACK_KVM_10163, + "failed to read backup file %s: %s", to.getPath(), to.getError()) + .withOpaque("path", to.getPath())); + continue; + } + if (to.getContentBase64() == null) { + errors.add(operr(ORG_ZSTACK_KVM_10163, "backup file %s returns empty content", to.getPath()) + .withOpaque("path", to.getPath())); + continue; + } + + VmHostFileType type = VmHostFileType.valueOf(to.getType()); + + VmHostBackupFileVO backupFile = new VmHostBackupFileVO(); + backupFile.setUuid(Platform.getUuid()); + backupFile.setResourceUuid(msg.getBackupResourceUuid()); + backupFile.setType(type); + backupFile.setCreateDate(now); + backupFile.setLastOpDate(now); + backupFilesToPersist.add(backupFile); + + VmHostFileContentVO content = new VmHostFileContentVO(); + content.setUuid(backupFile.getUuid()); + content.setContent(Base64.getDecoder().decode(to.getContentBase64())); + content.setFormat(VmHostFileContentFormat.valueOf(to.getFileFormat())); + content.setCreateDate(now); + content.setLastOpDate(now); + contentsToPersist.add(content); + + VmHostFileVO sourceFile = findOneOrNull(sourceHostFiles, item -> item.getType() == type); + if (sourceFile != null) { + backupFromMap.put(backupFile, sourceFile); + } + } + + new SQLBatch() { + @Override + protected void scripts() { + Set backupUuidSet = new HashSet<>(); + for (VmHostBackupFileVO bf : backupFilesToPersist) { + backupUuidSet.addAll(q(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, bf.getResourceUuid()) + .eq(VmHostBackupFileVO_.type, bf.getType()) + .select(VmHostBackupFileVO_.uuid) + .listValues()); + } + + if (!backupUuidSet.isEmpty()) { + // TODO: need refactor -> use VmHostBackupFileDeletionMsg + safeForEach(backupUuidSet, resourceKeyBackend::cleanEncryptedResourceKey); + sql(VmHostBackupFileVO.class) + .in(VmHostBackupFileVO_.uuid, backupUuidSet) + .delete(); + } + + if (!backupFilesToPersist.isEmpty()) { + databaseFacade.persistCollection(backupFilesToPersist); + } + if (!contentsToPersist.isEmpty()) { + databaseFacade.persistCollection(contentsToPersist); + } + } + }.execute(); + + for (VmHostBackupFileVO backup : backupFilesToPersist) { + VmHostFileVO source = backupFromMap.get(backup); + if (source != null) { + try { + vmHostFileFactory.createBackupBase(backup).afterBackup(source); + } catch (Exception e) { + logger.warn(String.format("failed to execute afterBackup hook for VmHostBackupFileVO[uuid:%s, type:%s]: %s", + backup.getUuid(), backup.getType(), e.getMessage()), e); + } + } + } + + if (errors.isEmpty()) { + return null; + } + + return operr(ORG_ZSTACK_KVM_10163, "failed to read backup file content from host[uuid=%s]", msg.getHostUuid()) + .causedBy(errors); + } + + @SuppressWarnings("rawtypes") + private void handle(CloneVmHostFileMsg msg) { + CloneVmHostFileReply reply = new CloneVmHostFileReply(); + + final Set types = vmHostFileFactory.vmHostFileTypeNeedRegisterForVm(msg.getSrcVmUuid()); + if (types.isEmpty()) { + bus.reply(msg, reply); + return; + } + + CloneVmHostFileContext context = new CloneVmHostFileContext(); + if (types.contains(VmHostFileType.NvRam)) { + context.typesNeedClone.add(VmHostFileType.NvRam); + } + + if (types.contains(VmHostFileType.TpmState)) { + boolean resetTpm; + if (msg.getResetTpm() == null) { + ResourceConfig resourceConfig = resourceConfigFacade.getResourceConfig(RESET_TPM_AFTER_VM_CLONE.getIdentity()); + resetTpm = resourceConfig.getResourceConfigValue(msg.getSrcVmUuid(), Boolean.class); + } else { + resetTpm = msg.getResetTpm(); + } + if (!resetTpm) { + context.typesNeedClone.add(VmHostFileType.TpmState); + } + } + logger.debug(String.format("clone VM[uuid=%s] host files for types: %s", msg.getSrcVmUuid(), context.typesNeedClone)); + + SimpleFlowChain chain = new SimpleFlowChain(); + chain.setName("clone-vm-host-file"); + chain.then(new NoRollbackFlow() { + String __name__ = "prepare-sync-vm-host-file-context-list"; + + @Override + public void run(FlowTrigger trigger, Map data) { + for (VmHostFileType type : context.typesNeedClone) { + VmHostFileVO file = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, msg.getSrcVmUuid()) + .eq(VmHostFileVO_.type, type) + .orderByDesc(VmHostFileVO_.lastSyncDate) + .limit(1) + .find(); + if (file == null) { + logger.debug(String.format("skip to read/write %s host file for VM[vmUuid=%s]: file is not registered in MN", + type, msg.getSrcVmUuid())); + continue; + } + context.files.add(file); + } + + if (context.files.isEmpty()) { + trigger.next(); + return; + } + + VmInstanceState vmState = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, msg.getSrcVmUuid()) + .select(VmInstanceVO_.state) + .findValue(); + if (vmState == VmInstanceState.Stopped) { + boolean anyChange = context.files.stream() + .anyMatch(it -> it.getChangeDate() != null); + if (!anyChange) { + trigger.next(); + return; + } + } + + Map contextMap = new HashMap<>(); + for (VmHostFileVO file : context.files) { + contextMap.computeIfAbsent(file.getHostUuid(), hostUuid -> { + SyncVmHostFilesFromHostMsg syncContext = new SyncVmHostFilesFromHostMsg(); + syncContext.setHostUuid(hostUuid); + syncContext.setVmUuid(msg.getSrcVmUuid()); + syncContext.setSyncReason(PostClone.reason()); + return syncContext; + }); + } + context.syncContexts.addAll(contextMap.values()); + + for (VmHostFileVO file : context.files) { + SyncVmHostFilesFromHostMsg syncContext = contextMap.get(file.getHostUuid()); + if (file.getType() == VmHostFileType.NvRam) { + syncContext.setNvRamPath(file.getPath()); + } else if (file.getType() == VmHostFileType.TpmState) { + syncContext.setTpmStateFolder(file.getPath()); + } else { + throw new CloudRuntimeException("unsupported vm host file type: " + file.getType()); + } + } + + trigger.next(); + } + }).then(new NoRollbackFlow() { + String __name__ = "read-vm-host-file-from-origin-host"; + + @Override + public boolean skip(Map data) { + return context.syncContexts.isEmpty(); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + new While<>(context.syncContexts).each((syncContext, whileContext) -> { + bus.makeLocalServiceId(syncContext, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncContext, new CloudBusCallBack(whileContext) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + whileContext.addError(reply.getError()); + } + whileContext.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList == null || errorCodeList.getCauses().isEmpty()) { + trigger.next(); + return; + } + String details = String.join("\n", transform(errorCodeList.getCauses(), ErrorCode::getReadableDetails)); + if (msg.isIgnoreSyncError()) { + logger.warn(String.format( + "failed to sync host file for VM[uuid=%s] but ignoreSyncError=true, continue:\n%s", + msg.getSrcVmUuid(), details)); + trigger.next(); + } else { + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "failed to sync host file for VM[uuid=%s]", msg.getSrcVmUuid()) + .causedBy(errorCodeList)); + } + } + }); + } + }).then(new NoRollbackFlow() { + String __name__ = "determine-content-uuid"; + + @Override + public void run(FlowTrigger trigger, Map data) { + List missingTypes = new ArrayList<>(context.typesNeedClone); + missingTypes.removeAll(transform(context.files, VmHostFileVO::getType)); + if (missingTypes.isEmpty()) { + trigger.next(); + return; + } + + context.backupFiles.addAll(Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, msg.getSrcVmUuid()) + .in(VmHostBackupFileVO_.type, missingTypes) + .list()); + trigger.next(); + } + }).then(new NoRollbackFlow() { + String __name__ = "copy-host-content-database"; + + @Override + public boolean skip(Map data) { + return context.files.isEmpty() && context.backupFiles.isEmpty(); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + List uuidList = transform(context.files, VmHostFileVO::getUuid); + List filesAfterSyncing = Q.New(VmHostFileVO.class) + .in(VmHostFileVO_.uuid, uuidList) + .list(); + backupVmHostFile(filesAfterSyncing, context.backupFiles, msg.getDstVmUuidList()); + trigger.next(); + } + }).done(new FlowDoneHandler(msg) { + @Override + public void handle(Map data) { + bus.reply(msg, reply); + } + }).error(new FlowErrorHandler(msg) { + @Override + public void handle(ErrorCode errCode, Map data) { + reply.setError(errCode); + bus.reply(msg, reply); + } + }).start(); + } + + private void handle(BackupVmHostFileMsg msg) { + BackupVmHostFileReply reply = new BackupVmHostFileReply(); + List filesNeedPersists = backupVmHostFile( + msg.getVmUuid(), msg.getHostUuid(), msg.getToResourceUuidList()); + reply.setBackupFileUuidList(transform(filesNeedPersists, VmHostBackupFileVO::getUuid)); + bus.reply(msg, reply); + } + + private List backupVmHostFile(String fromVmUuid, String hostUuid, List toResourceList) { + List hostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, fromVmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .list(); + + if (hostFiles.isEmpty()) { + return new ArrayList<>(); + } + return backupVmHostFile(hostFiles, new ArrayList<>(), toResourceList); + } + + private List backupVmHostFile(List fileList, List backupFiles, List toResourceList) { + List uuidList = transform(fileList, VmHostFileVO::getUuid); + uuidList.addAll(transform(backupFiles, VmHostBackupFileVO::getUuid)); + List contents = Q.New(VmHostFileContentVO.class) + .in(VmHostFileContentVO_.uuid, uuidList) + .list(); + + List filesNeedPersists = new ArrayList<>(); + List contentsNeedPersists = new ArrayList<>(); + // value is VmHostBackupFileVO or VmHostFileVO + Map backupFromMap = new HashMap<>(); + + Timestamp now = Timestamp.from(Instant.now()); + for (String resourceUuid : toResourceList) { + for (String uuid : uuidList) { + VmHostFileContentVO srcContent = findOneOrNull(contents, + item -> item.getUuid().equals(uuid)); + if (srcContent == null) { + continue; + } + + VmHostFileVO vmHostFile = findOneOrNull(fileList, + item -> item.getUuid().equals(uuid)); + VmHostBackupFileVO vmHostBackupFile = vmHostFile == null ? + findOneOrNull(backupFiles, item -> item.getUuid().equals(uuid)) : null; + DebugUtils.Assert(vmHostFile != null || vmHostBackupFile != null, + "vmHostFile or vmHostBackupFile cannot be null"); + + VmHostBackupFileVO file = new VmHostBackupFileVO(); + file.setUuid(Platform.getUuid()); + file.setResourceUuid(resourceUuid); + file.setType(vmHostFile == null ? vmHostBackupFile.getType() : vmHostFile.getType()); + file.setCreateDate(now); + file.setLastOpDate(now); + filesNeedPersists.add(file); + backupFromMap.put(file, vmHostFile == null ? vmHostBackupFile : vmHostFile); + + VmHostFileContentVO content = new VmHostFileContentVO(); + content.setUuid(file.getUuid()); + content.setContent(srcContent.getContent()); + content.setFormat(srcContent.getFormat()); + content.setCreateDate(now); + content.setLastOpDate(now); + contentsNeedPersists.add(content); + } + } + + if (logger.isTraceEnabled()) { + logger.trace(String.format("persist VmHostFileContentVO [uuid=%s]", + transform(contentsNeedPersists, VmHostFileContentVO::getUuid))); + } + + new SQLBatch() { + @Override + protected void scripts() { + Set backupUuidSet = new HashSet<>(); + for (VmHostBackupFileVO bf : filesNeedPersists) { + backupUuidSet.addAll(q(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, bf.getResourceUuid()) + .eq(VmHostBackupFileVO_.type, bf.getType()) + .select(VmHostBackupFileVO_.uuid) + .listValues()); + } + + if (!backupUuidSet.isEmpty()) { + // TODO: need refactor -> use VmHostBackupFileDeletionMsg + safeForEach(backupUuidSet, resourceKeyBackend::cleanEncryptedResourceKey); + sql(VmHostBackupFileVO.class) + .in(VmHostBackupFileVO_.uuid, backupUuidSet) + .delete(); + } + + if (!filesNeedPersists.isEmpty()) { + databaseFacade.persistCollection(filesNeedPersists); + } + if (!contentsNeedPersists.isEmpty()) { + databaseFacade.persistCollection(contentsNeedPersists); + } + } + }.execute(); + + for (VmHostBackupFileVO backup : filesNeedPersists) { + final Object backupFrom = backupFromMap.get(backup); + try { + if (backupFrom instanceof VmHostBackupFileVO) { + vmHostFileFactory.createBackupBase(backup).afterBackup((VmHostBackupFileVO) backupFrom); + } else if (backupFrom instanceof VmHostFileVO) { + vmHostFileFactory.createBackupBase(backup).afterBackup((VmHostFileVO) backupFrom); + } + } catch (Exception e) { + logger.warn(String.format("failed to execute afterBackup hook for VmHostBackupFileVO[uuid:%s, type:%s]: %s", + backup.getUuid(), backup.getType(), e.getMessage()), e); + } + } + + return filesNeedPersists; + } + + private void handle(RestoreVmHostFileMsg msg) { + RestoreVmHostFileReply reply = new RestoreVmHostFileReply(); + + List backupFiles = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, msg.getSnapshotGroupUuid()) + .list(); + + Tuple tuple = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid, VmInstanceVO_.lastHostUuid) + .eq(VmInstanceVO_.uuid, msg.getVmInstanceUuid()) + .findTuple(); + if (tuple == null) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "VM instance [uuid:%s] not found", msg.getVmInstanceUuid())); + bus.reply(msg, reply); + return; + } + + String hostUuid = tuple.get(0, String.class); + if (hostUuid == null) { + hostUuid = tuple.get(1, String.class); + } + if (hostUuid == null) { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "VM instance [uuid:%s] has no host", msg.getVmInstanceUuid())); + bus.reply(msg, reply); + return; + } + + List currentHostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, msg.getVmInstanceUuid()) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .list(); + + Map currentFilesByType = new HashMap<>(); + for (VmHostFileVO file : currentHostFiles) { + currentFilesByType.put(file.getType(), file); + } + + Map backupFilesByType = new HashMap<>(); + for (VmHostBackupFileVO file : backupFiles) { + backupFilesByType.put(file.getType(), file); + } + + Set allTypes = new HashSet<>(); + allTypes.addAll(currentFilesByType.keySet()); + allTypes.addAll(backupFilesByType.keySet()); + + if (allTypes.isEmpty()) { + bus.reply(msg, reply); + return; + } + + // Batch query all backup file content before loop + List backupUuids = transform(backupFiles, VmHostBackupFileVO::getUuid); + Map backupContentMap = new HashMap<>(); + if (!backupUuids.isEmpty()) { + List backupContents = Q.New(VmHostFileContentVO.class) + .in(VmHostFileContentVO_.uuid, backupUuids) + .list(); + backupContentMap.putAll(toMap(backupContents, VmHostFileContentVO::getUuid, Function.identity())); + } + + List fileList = new ArrayList<>(); + for (VmHostFileType type : allTypes) { + VmHostFileTO to = new VmHostFileTO(); + to.setType(type.toString()); + + boolean hasCurrentFile = currentFilesByType.containsKey(type); + boolean hasBackupFile = backupFilesByType.containsKey(type); + + if (hasBackupFile) { + // Write operation + VmHostBackupFileVO backupFile = backupFilesByType.get(type); + VmHostFileContentVO content = backupContentMap.get(backupFile.getUuid()); + if (content == null) { + logger.warn(String.format("backup file content [uuid:%s] not found for type %s", + backupFile.getUuid(), type)); + continue; + } + + to.setPath(buildPathForVmHostFileType(type, msg.getVmInstanceUuid())); + to.setFileFormat(content.getFormat().toString()); + to.setOperation(VmHostFileOperation.Write.toString()); + String contentBase64 = Base64.getEncoder().encodeToString(content.getContent()); + to.setContentBase64(contentBase64); + + fileList.add(to); + } else if (hasCurrentFile) { + // Delete operation + VmHostFileVO currentFile = currentFilesByType.get(type); + to.setPath(currentFile.getPath()); + to.setOperation(VmHostFileOperation.Delete.toString()); + + fileList.add(to); + } + } + + if (fileList.isEmpty()) { + bus.reply(msg, reply); + return; + } + + final String finalHostUuid = hostUuid; + SimpleFlowChain.of("restore-vm-host-file") + .then(Flow.of("send-cmd") + .handle(trigger -> { + KVMAgentCommands.WriteVmHostFileContentCmd cmd = new KVMAgentCommands.WriteVmHostFileContentCmd(); + cmd.setHostFiles(fileList); + + KvmCommandSender sender = new KvmCommandSender(finalHostUuid); + sender.send(cmd, WRITE_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.WriteVmHostFileContentResponse writeRsp = wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class); + return writeRsp.isSuccess() ? null : + operr(ORG_ZSTACK_KVM_10163, + "failed to write/delete host file response: %s", writeRsp.getError()); + }, new ReturnValueCompletion(trigger) { + @Override + public void success(KvmResponseWrapper wrapper) { + logger.info(String.format("success to restore host files for VM[uuid:%s] from snapshot group[uuid:%s]", + msg.getVmInstanceUuid(), msg.getSnapshotGroupUuid())); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "failed to restore host files for VM[uuid:%s]", msg.getVmInstanceUuid()) + .causedBy(errorCode)); + } + }); + }) + .build()) + .then(Flow.of("persist-content-in-db") + .handle(trigger -> { + Timestamp now = Timestamp.from(Instant.now()); + + List allUuids = new ArrayList<>(); + allUuids.addAll(transform(backupFilesByType.values(), VmHostBackupFileVO::getUuid)); + allUuids.addAll(transform(currentFilesByType.values(), VmHostFileVO::getUuid)); + + Map contentMap = new HashMap<>(); + if (!allUuids.isEmpty()) { + List contents = Q.New(VmHostFileContentVO.class) + .in(VmHostFileContentVO_.uuid, allUuids) + .list(); + contentMap.putAll(toMap(contents, VmHostFileContentVO::getUuid, Function.identity())); + } + + for (VmHostFileType type : allTypes) { + boolean hasCurrentFile = currentFilesByType.containsKey(type); + boolean hasBackupFile = backupFilesByType.containsKey(type); + + if (hasBackupFile) { + VmHostBackupFileVO backupFile = backupFilesByType.get(type); + VmHostFileContentVO backupContent = contentMap.get(backupFile.getUuid()); + if (backupContent == null) { + continue; + } + + if (hasCurrentFile) { + // update existing VmHostFileVO and VmHostFileContentVO + VmHostFileVO currentFile = currentFilesByType.get(type); + SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.uuid, currentFile.getUuid()) + .set(VmHostFileVO_.lastSyncReason, Restore.reason(msg.getSyncReason())) + .set(VmHostFileVO_.lastOpDate, now) + .set(VmHostFileVO_.lastSyncDate, now) + .update(); + + VmHostFileContentVO existingContent = contentMap.get(currentFile.getUuid()); + if (existingContent != null) { + SQL.New(VmHostFileContentVO.class) + .eq(VmHostFileContentVO_.uuid, currentFile.getUuid()) + .set(VmHostFileContentVO_.content, backupContent.getContent()) + .set(VmHostFileContentVO_.format, backupContent.getFormat()) + .set(VmHostFileContentVO_.lastOpDate, now) + .update(); + } else { + VmHostFileContentVO newContent = new VmHostFileContentVO(); + newContent.setUuid(currentFile.getUuid()); + newContent.setContent(backupContent.getContent()); + newContent.setFormat(backupContent.getFormat()); + newContent.setCreateDate(now); + newContent.setLastOpDate(now); + databaseFacade.persist(newContent); + } + } else { + // create new VmHostFileVO and VmHostFileContentVO + VmHostFileVO newFile = new VmHostFileVO(); + newFile.setUuid(Platform.getUuid()); + newFile.setResourceName(String.format("%s file for %s", type, msg.getVmInstanceUuid())); + newFile.setVmInstanceUuid(msg.getVmInstanceUuid()); + newFile.setHostUuid(finalHostUuid); + newFile.setType(type); + newFile.setPath(buildPathForVmHostFileType(type, msg.getVmInstanceUuid())); + newFile.setLastSyncReason(Restore.reason(msg.getSyncReason())); + newFile.setLastSyncDate(now); + newFile.setCreateDate(now); + newFile.setLastOpDate(now); + databaseFacade.persist(newFile); + + VmHostFileContentVO newContent = new VmHostFileContentVO(); + newContent.setUuid(newFile.getUuid()); + newContent.setContent(backupContent.getContent()); + newContent.setFormat(backupContent.getFormat()); + newContent.setCreateDate(now); + newContent.setLastOpDate(now); + databaseFacade.persist(newContent); + } + } else if (hasCurrentFile) { + // delete VmHostFileVO and VmHostFileContentVO + VmHostFileVO currentFile = currentFilesByType.get(type); + SQL.New(VmHostFileContentVO.class) + .eq(VmHostFileContentVO_.uuid, currentFile.getUuid()) + .delete(); + SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.uuid, currentFile.getUuid()) + .delete(); + } + } + + trigger.next(); + }) + .build()) + .propagateExceptionTo(msg) + .done(() -> bus.reply(msg, reply)) + .error(errorCode -> { + reply.setError(errorCode); + bus.reply(msg, reply); + }) + .start(); + } + + private void handle(BackupVmHostFileOnHypervisorMsg msg) { + KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid()); + + KVMAgentCommands.BackupVmHostFileCmd cmd = new KVMAgentCommands.BackupVmHostFileCmd(); + cmd.setVmHostFileBackupJobs(msg.getVmHostFileBackupJobs()); + + BackupVmHostFileOnHypervisorReply reply = new BackupVmHostFileOnHypervisorReply(); + sender.send(cmd, BACKUP_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.BackupVmHostFileResponse rsp = wrapper.getResponse(KVMAgentCommands.BackupVmHostFileResponse.class); + return rsp.isSuccess() ? null : + operr(ORG_ZSTACK_KVM_10163, "failed to backup vm host file on hypervisor[hostUuid=%s]", msg.getHostUuid()) + .withOpaque("host.uuid", msg.getHostUuid()) + .withOpaque("agent.error", rsp.getError()); + }, new ReturnValueCompletion(msg) { + @Override + public void success(KvmResponseWrapper wrapper) { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(VmHostFileDeletionMsg msg) { + VmHostFileDeletionReply reply = new VmHostFileDeletionReply(); + databaseFacade.removeByPrimaryKey(msg.getUuid(), VmHostFileVO.class); + bus.reply(msg, reply); + } + + private void handle(VmHostBackupFileDeletionMsg msg) { + VmHostBackupFileDeletionReply reply = new VmHostBackupFileDeletionReply(); + VmHostBackupFileVO backup = databaseFacade.findByUuid(msg.getUuid(), VmHostBackupFileVO.class); + if (backup != null) { + try { + vmHostFileFactory.createBackupBase(backup).clean(); + } catch (Exception e) { + if (msg.isForceDelete()) { + logger.warn(String.format("force delete VmHostBackupFile[uuid:%s] ignored cleanup error: %s", + backup.getUuid(), e.getMessage()), e); + } else { + reply.setError(operr(ORG_ZSTACK_KVM_10163, "failed to delete VmHostBackupFile[uuid:%s]: %s", + backup.getUuid(), e.getMessage())); + } + } + } + bus.reply(msg, reply); + } + + @Override + public void cleanVmHostBackupFile(String resourceUuid) { + List backups = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, resourceUuid) + .list(); + + for (VmHostBackupFileVO backup : backups) { + try { + vmHostFileFactory.createBackupBase(backup).clean(); + } catch (Exception e) { + logger.warn(String.format("failed to delete VmHostBackupFileVO[uuid:%s, type:%s]: %s", + backup.getUuid(), backup.getType(), e.getMessage()), e); + } + } + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmVmHostFileFactory.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmVmHostFileFactory.java new file mode 100644 index 00000000000..98cedff3612 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmVmHostFileFactory.java @@ -0,0 +1,51 @@ +package org.zstack.kvm.efi; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.devices.VmTpmManager; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.kvm.tpm.TpmStateVmHostBackupFileBase; +import org.zstack.kvm.tpm.TpmStateVmHostFileBase; +import org.zstack.kvm.vmfiles.AbstractVmHostBackupFileBase; +import org.zstack.kvm.vmfiles.AbstractVmHostFileBase; +import org.zstack.resourceconfig.ResourceConfigFacade; + +import java.util.Set; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class KvmVmHostFileFactory { + @Autowired + private ResourceConfigFacade resourceConfigFacade; + @Autowired + private VmTpmManager vmTpmManager; + + public AbstractVmHostFileBase createBase(VmHostFileVO file) { + switch (file.getType()) { + case NvRam: return new NvRamVmHostFileBase(file); + case TpmState: return new TpmStateVmHostFileBase(file); + default: throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10163, + "invalid VM host file type: %s", file.getType())); + } + } + + public AbstractVmHostBackupFileBase createBackupBase(VmHostBackupFileVO backupFile) { + switch (backupFile.getType()) { + case NvRam: return new NvRamVmHostBackupFileBase(backupFile); + case TpmState: return new TpmStateVmHostBackupFileBase(backupFile); + default: throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10163, + "invalid VM host file type: %s", backupFile.getType())); + } + } + + public Set vmHostFileTypeNeedRegisterForVm(String vmUuid) { + return vmTpmManager.vmHostFileTypeNeedRegisterForVm(vmUuid); + } + + public boolean needRegister(VmHostFileType type, String vmUuid) { + return vmTpmManager.needRegister(type, vmUuid); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/NvRamVmHostBackupFileBase.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/NvRamVmHostBackupFileBase.java new file mode 100644 index 00000000000..8d78b9ac0d6 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/NvRamVmHostBackupFileBase.java @@ -0,0 +1,16 @@ +package org.zstack.kvm.efi; + +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.kvm.vmfiles.AbstractVmHostBackupFileBase; + +public class NvRamVmHostBackupFileBase extends AbstractVmHostBackupFileBase { + public NvRamVmHostBackupFileBase(VmHostBackupFileVO self) { + super(self); + } + + @Override + public VmHostFileType type() { + return VmHostFileType.NvRam; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/NvRamVmHostFileBase.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/NvRamVmHostFileBase.java new file mode 100644 index 00000000000..cf539ee2278 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/NvRamVmHostFileBase.java @@ -0,0 +1,16 @@ +package org.zstack.kvm.efi; + +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.kvm.vmfiles.AbstractVmHostFileBase; + +public class NvRamVmHostFileBase extends AbstractVmHostFileBase { + public NvRamVmHostFileBase(VmHostFileVO self) { + super(self); + } + + @Override + public VmHostFileType type() { + return VmHostFileType.NvRam; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java new file mode 100644 index 00000000000..534a08ab0be --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java @@ -0,0 +1,898 @@ +package org.zstack.kvm.tpm; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.VmGlobalConfig; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend; +import org.zstack.compute.vm.devices.VmTpmManager; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend.CloneEncryptedResourceKeyContext; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.core.workflow.SimpleFlowChain; +import org.zstack.header.core.Completion; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.core.workflow.FlowDoneHandler; +import org.zstack.header.core.workflow.FlowErrorHandler; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowRollback; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.core.NoErrorCompletion; +import org.zstack.header.host.HostConstant; +import org.zstack.header.message.MessageReply; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager.GetOrCreateResourceKeyContext; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager.ResourceKeyResult; +import org.zstack.header.secret.SecretHostDeleteMsg; +import org.zstack.header.secret.SecretHostDefineMsg; +import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.header.secret.SecretHostGetMsg; +import org.zstack.header.secret.SecretHostGetReply; +import org.zstack.header.secret.ResolveVtpmLibvirtSecretOnHypervisorMsg; +import org.zstack.header.secret.ResolveVtpmLibvirtSecretOnHypervisorReply; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_; +import org.zstack.header.tpm.entity.TpmSpec; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.tpm.message.TpmDeletionMsg; +import org.zstack.header.vm.HaStartVmInstanceMsg; +import org.zstack.header.vm.PreVmInstantiateResourceExtensionPoint; +import org.zstack.header.vm.VmAfterExpungeExtensionPoint; +import org.zstack.header.vm.VmJustBeforeDeleteFromDbExtensionPoint; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceMigrateExtensionPoint; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.VmInstantiateResourceException; +import org.zstack.header.vm.VmStateChangedExtensionPoint; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.header.vm.devices.VmDevicesSpec; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMHostInventory; +import org.zstack.kvm.KVMStartVmExtensionPoint; +import org.zstack.kvm.efi.KvmSecureBootExtensions; +import org.zstack.kvm.efi.KvmSecureBootExtensions.*; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import static org.zstack.header.tpm.TpmConstants.SERVICE_ID; +import static org.zstack.kvm.KVMConstant.*; +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class KvmTpmExtensions implements KVMStartVmExtensionPoint, + PreVmInstantiateResourceExtensionPoint, + VmInstanceMigrateExtensionPoint, + VmAfterExpungeExtensionPoint, + VmStateChangedExtensionPoint, + VmJustBeforeDeleteFromDbExtensionPoint { + private static final CLogger logger = Utils.getLogger(KvmTpmExtensions.class); + + @Autowired + private KvmSecureBootExtensions secureBootExtensions; + @Autowired + private DatabaseFacade databaseFacade; + @Autowired + private TpmEncryptedResourceKeyBackend resourceKeyBackend; + @Autowired + private EncryptedResourceKeyManager resourceKeyManager; + @Autowired + private CloudBus bus; + + private final Object hostFileLock = new Object(); + private final Map volumeMigratingSourceHostCache = new ConcurrentHashMap<>(); + + @Override + public void beforeStartVmOnKvm(KVMHostInventory host, VmInstanceSpec spec, KVMAgentCommands.StartVmCmd cmd) { + final VmDevicesSpec devicesSpec = spec.getDevicesSpec(); + if (devicesSpec == null || devicesSpec.getTpm() == null || !devicesSpec.getTpm().isEnable()) { + return; + } + + String keyProviderUuid = devicesSpec.getTpm().getKeyProviderUuid(); + if (StringUtils.isBlank(keyProviderUuid)) { + keyProviderUuid = resourceKeyBackend.findKeyProviderUuidByTpm(devicesSpec.getTpm().getTpmUuid()); + } + + TpmTO tpm = new TpmTO(); + tpm.setKeyProviderUuid(keyProviderUuid); + tpm.setSecretUuid(devicesSpec.getTpm().getSecretUuid()); + tpm.setInstallPath(buildTpmStateFilePath(cmd.getVmInstanceUuid())); + cmd.setTpm(tpm); + + synchronized (hostFileLock) { + VmHostFileVO tpmStateFile = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, cmd.getVmInstanceUuid()) + .eq(VmHostFileVO_.type, VmHostFileType.TpmState) + .eq(VmHostFileVO_.hostUuid, host.getUuid()) + .find(); + if (tpmStateFile == null) { + tpmStateFile = new VmHostFileVO(); + tpmStateFile.setUuid(Platform.getUuid()); + tpmStateFile.setHostUuid(host.getUuid()); + tpmStateFile.setVmInstanceUuid(cmd.getVmInstanceUuid()); + tpmStateFile.setType(VmHostFileType.TpmState); + tpmStateFile.setPath(tpm.getInstallPath()); + tpmStateFile.setCreateDate(Timestamp.from(Instant.now())); + tpmStateFile.setResourceName("TpmState file for " + cmd.getVmInstanceUuid()); + databaseFacade.persist(tpmStateFile); + } else { + SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.uuid, tpmStateFile.getUuid()) + .set(VmHostFileVO_.path, tpm.getInstallPath()) + .set(VmHostFileVO_.lastOpDate, Timestamp.from(Instant.now())) + .update(); + } + } + } + + @Override + public void startVmOnKvmSuccess(KVMHostInventory host, VmInstanceSpec spec) { + if (spec.getMessage() instanceof HaStartVmInstanceMsg) { + String vmUuid = spec.getVmInventory() == null ? null : spec.getVmInventory().getUuid(); + String srcHostUuid = spec.getVmInventory() == null ? null : spec.getVmInventory().getLastHostUuid(); + Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid); + boolean vmIsOnDestHost = isVmCurrentlyOnExpectedHost(vmUuid, host.getUuid()); + if (vmIsOnDestHost && StringUtils.isNotBlank(srcHostUuid) && !host.getUuid().equals(srcHostUuid)) { + deleteHostSecretBestEffort(srcHostUuid, vmUuid, keyVersion, + "ha-start-success"); + } + } + clearRollbackInfo(spec); + } + + @Override + public void startVmOnKvmFailed(KVMHostInventory host, VmInstanceSpec spec, ErrorCode err) { + clearRollbackInfo(spec); + } + + @Override + public void preBeforeInstantiateVmResource(VmInstanceSpec spec) throws VmInstantiateResourceException { + // do-nothing + } + + @Override + @SuppressWarnings("rawtypes") + public void preInstantiateVmResource(VmInstanceSpec spec, Completion completion) { + final VmDevicesSpec devicesSpec = spec.getDevicesSpec(); + if (devicesSpec == null || devicesSpec.getTpm() == null || !devicesSpec.getTpm().isEnable()) { + completion.success(); + return; + } + + TpmSpec tpmSpec = devicesSpec.getTpm(); + clearRollbackInfo(spec); + final PrepareTpmResourceContext context = new PrepareTpmResourceContext(); + context.tpmUuid = tpmSpec.getTpmUuid(); + context.backupFileUuid = tpmSpec.getBackupFileUuid(); // maybe null + context.providerUuid = resourceKeyBackend.findKeyProviderUuidByTpm(context.tpmUuid); + if (StringUtils.isBlank(context.providerUuid) && StringUtils.isNotBlank(tpmSpec.getKeyProviderUuid())) { + int updated = resourceKeyBackend.applyKeyProviderWithKek(context.tpmUuid, tpmSpec.getKeyProviderUuid()); + if (updated > 0) { + context.providerUuid = tpmSpec.getKeyProviderUuid(); + logger.info(String.format( + "auto repaired TPM key provider binding for tpm[uuid:%s], providerUuid:%s", + context.tpmUuid, context.providerUuid)); + } else if (!resourceKeyBackend.checkTpmKeyProviderAttached(context.tpmUuid)) { + context.providerUuid = tpmSpec.getKeyProviderUuid(); + resourceKeyBackend.attachKeyProviderToTpm(context.tpmUuid, context.providerUuid); + logger.info(String.format( + "auto repaired TPM key provider binding for tpm[uuid:%s], provider[uuid:%s]", + context.tpmUuid, context.providerUuid)); + } else { + logger.warn(String.format( + "failed in-place providerUuid repair in preInstantiate for tpm[uuid:%s], providerUuid:%s, existing ref rows remain", + context.tpmUuid, tpmSpec.getKeyProviderUuid())); + } + } + context.keyVersion = resourceKeyBackend.findKeyVersionByTpm(context.tpmUuid); + context.instantiateForNewVm = spec.getCurrentVmOperation() == VmInstanceConstant.VmOperation.NewCreate; + context.enableKeyProvider = !VmGlobalConfig.ALLOWED_TPM_VM_WITHOUT_KMS.value(Boolean.class); + + final SimpleFlowChain chain = new SimpleFlowChain(); + chain.setName("prepare-tpm-resources-for-vm-" + spec.getVmInventory().getUuid()); + chain.then(new NoRollbackFlow() { + String __name__ = "prepare-tpm-state-file-on-host"; + + @Override + public void run(FlowTrigger trigger, Map data) { + PrepareHostFileContext innerContext = new PrepareHostFileContext(); + innerContext.hostUuid = spec.getDestHost().getUuid(); + innerContext.vmUuid = spec.getVmInventory().getUuid(); + innerContext.type = VmHostFileType.TpmState; + innerContext.backupUuid = context.backupFileUuid; + innerContext.syncReason = "pre-instantiate VM resource"; + secureBootExtensions.prepareHostFileOnHost(innerContext, new Completion(trigger) { + @Override + public void success() { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + }).then(new Flow() { + String __name__ = "clone-tpm-resource-key-from-snapshot-source"; + + @Override + public boolean skip(Map data) { + return !context.enableKeyProvider; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + String srcTpmUuid = findSourceTpmUuidFromSnapshotTpmBackupFile(context.backupFileUuid); + if (StringUtils.isBlank(srcTpmUuid)) { + trigger.next(); + return; + } + CloneEncryptedResourceKeyContext cloneCtx = new CloneEncryptedResourceKeyContext(); + cloneCtx.srcTpmUuid = srcTpmUuid; + cloneCtx.dstTpmUuid = context.tpmUuid; + cloneCtx.resetTpm = false; + resourceKeyBackend.cloneEncryptedResourceKey(cloneCtx, new Completion(trigger) { + @Override + public void success() { + context.providerUuid = resourceKeyBackend.findKeyProviderUuidByTpm(context.tpmUuid); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + + // Use clone op above and will not set rollback flag TpmSpec.resourceKeyCreatedNew + // to true, so use flow rollback instead preReleaseVmResource rollback. And we + // definitely don't need to delete keytool secret on shapshot case. + @Override + public void rollback(FlowRollback trigger, Map data) { + if (StringUtils.isNotBlank(context.backupFileUuid)) { + try { + resourceKeyBackend.detachKeyProviderFromTpm(context.tpmUuid); + } catch (Exception e) { + logger.warn(String.format("failed to detach key provider ref for tpm[uuid:%s]: %s", + context.tpmUuid, e.getMessage())); + } + } + trigger.rollback(); + } + }).then(new NoRollbackFlow() { + String __name__ = "get-secret-on-host-first"; + + @Override + public boolean skip(Map data) { + return !context.enableKeyProvider; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + if (context.instantiateForNewVm && context.keyVersion == null) { + trigger.next(); + return; + } + + // NewCreate cloned from an existing TPM may already carry keyVersion; allow it. + // For non-NewCreate, keyVersion must exist (validated above). + SecretHostGetMsg innerMsg = new SecretHostGetMsg(); + innerMsg.setHostUuid(spec.getDestHost().getUuid()); + innerMsg.setVmUuid(spec.getVmInventory().getUuid()); + innerMsg.setPurpose("vtpm"); + innerMsg.setKeyVersion(context.keyVersion); + innerMsg.setUsageInstance(HOST_SECRET_USAGE_INSTANCE_VTPM); + bus.makeTargetServiceIdByResourceUuid(innerMsg, HostConstant.SERVICE_ID, innerMsg.getHostUuid()); + bus.send(innerMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + SecretHostGetReply r = reply.castReply(); + if (StringUtils.isBlank(r.getSecretUuid())) { + logger.warn(String.format( + "get secret on host succeeded but returned empty secretUuid for tpm[uuid:%s], host[uuid:%s], recreate DEK from KMS", + context.tpmUuid, spec.getDestHost().getUuid())); + trigger.next(); + return; + } + spec.getDevicesSpec().getTpm().setSecretUuid(r.getSecretUuid()); + context.vtpmSecretAlreadyOnHost = true; + trigger.next(); + return; + } + + ErrorCode errorCode = reply.getError(); + if (errorCode != null && isVtpmSecretNotFoundOnHost(errorCode)) { + trigger.next(); + return; + } + + trigger.fail(errorCode != null ? errorCode : operr(ORG_ZSTACK_KVM_10163, "get secret on host failed")); + } + }); + } + }).then(new NoRollbackFlow() { + String __name__ = "get-or-create-key-and-dek"; + + @Override + public boolean skip(Map data) { + return !context.enableKeyProvider || context.vtpmSecretAlreadyOnHost; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + GetOrCreateResourceKeyContext keyCtx = new GetOrCreateResourceKeyContext(); + keyCtx.setResourceUuid(context.tpmUuid); + keyCtx.setResourceType(TpmVO.class.getSimpleName()); + keyCtx.setKeyProviderUuid(context.providerUuid); + keyCtx.setPurpose("vtpm"); + + resourceKeyManager.getOrCreateKey(keyCtx, new ReturnValueCompletion(trigger) { + @Override + public void success(ResourceKeyResult result) { + tpmSpec.setResourceKeyCreatedNew(result.isCreatedNewKey()); + tpmSpec.setResourceKeyProviderUuid(result.getKeyProviderUuid()); + context.dekBase64 = result.getDekBase64(); + context.keyVersion = result.getKeyVersion(); + if (context.keyVersion == null) { + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "missing keyVersion for tpm[uuid:%s] after getOrCreateKey", context.tpmUuid)); + return; + } + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + }).then(new NoRollbackFlow() { + String __name__ = "define-secret-on-host"; + + @Override + public boolean skip(Map data) { + return !context.enableKeyProvider || context.vtpmSecretAlreadyOnHost; + } + + @Override + public void run(FlowTrigger trigger, Map data) { + if (context.dekBase64 == null) { + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "missing dekBase64 for tpm[uuid:%s] before define-secret-on-host", context.tpmUuid)); + return; + } + + SecretHostDefineMsg innerMsg = new SecretHostDefineMsg(); + innerMsg.setHostUuid(spec.getDestHost().getUuid()); + innerMsg.setVmUuid(spec.getVmInventory().getUuid()); + innerMsg.setDekBase64(context.dekBase64); + innerMsg.setPurpose("vtpm"); + innerMsg.setKeyVersion(context.keyVersion); + innerMsg.setUsageInstance(HOST_SECRET_USAGE_INSTANCE_VTPM); + innerMsg.setDescription("Define secret for VM " + spec.getVmInventory().getUuid()); + bus.makeTargetServiceIdByResourceUuid(innerMsg, HostConstant.SERVICE_ID, innerMsg.getHostUuid()); + bus.send(innerMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + SecretHostDefineReply r = reply.castReply(); + spec.getDevicesSpec().getTpm().setSecretUuid(r.getSecretUuid()); + context.clearSensitiveData(); + trigger.next(); + } else { + context.clearSensitiveData(); + trigger.fail(reply.getError()); + } + } + }); + } + }).done(new FlowDoneHandler(completion) { + @Override + public void handle(Map data) { + context.clearSensitiveData(); + completion.success(); + } + }).error(new FlowErrorHandler(completion) { + @Override + public void handle(ErrorCode errCode, Map data) { + context.clearSensitiveData(); + completion.fail(errCode); + } + }).start(); + } + + static class PrepareTpmStateHostFileContext { + String hostUuid; + String vmUuid; + + // whether the NvRam is on the same host as before + boolean sameHost = false; + VmHostFileVO tpmStateFile; + } + + static class PrepareTpmResourceContext { + boolean enableKeyProvider; + String tpmUuid; + String backupFileUuid; + String providerUuid; + Integer keyVersion; + String dekBase64; + boolean instantiateForNewVm; + boolean vtpmSecretAlreadyOnHost; + + void clearSensitiveData() { + dekBase64 = null; + } + } + + @Override + public void preReleaseVmResource(VmInstanceSpec spec, Completion completion) { + TpmSpec tpmSpec = spec.getDevicesSpec() == null ? null : spec.getDevicesSpec().getTpm(); + if (tpmSpec == null || !tpmSpec.isResourceKeyCreatedNew()) { + completion.success(); + return; + } + + ResourceKeyResult result = new ResourceKeyResult(); + result.setResourceUuid(tpmSpec.getTpmUuid()); + result.setResourceType(TpmVO.class.getSimpleName()); + result.setKeyProviderUuid(tpmSpec.getResourceKeyProviderUuid()); + result.setCreatedNewKey(true); + + resourceKeyManager.rollbackCreatedKey(result, new Completion(completion) { + @Override + public void success() { + clearRollbackInfo(spec); + completion.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("failed to rollback TPM resource key for tpm[uuid:%s]: %s", + tpmSpec.getTpmUuid(), errorCode != null ? errorCode.getDetails() : "")); + clearRollbackInfo(spec); + completion.success(); + } + }); + } + + @Override + public void vmJustBeforeDeleteFromDb(VmInstanceInventory inv) { + String tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, inv.getUuid()) + .select(TpmVO_.uuid) + .findValue(); + if (tpmUuid == null) { + return; + } + + // Delete host secrets while TPM row and key version are still resolvable. + // RemoveTpm may skip or fail (e.g. VM not in Stopped), and callback order may change. + Integer keyVersion = resourceKeyBackend.findKeyVersionByTpm(tpmUuid); + Set hostUuids = new HashSet<>(); + List tpmStateFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, inv.getUuid()) + .eq(VmHostFileVO_.type, VmHostFileType.TpmState) + .list(); + for (VmHostFileVO f : tpmStateFiles) { + if (StringUtils.isNotBlank(f.getHostUuid())) { + hostUuids.add(f.getHostUuid()); + } + } + if (StringUtils.isNotBlank(inv.getHostUuid())) { + hostUuids.add(inv.getHostUuid()); + } + if (StringUtils.isNotBlank(inv.getLastHostUuid())) { + hostUuids.add(inv.getLastHostUuid()); + } + for (String hostUuid : hostUuids) { + deleteHostSecretBestEffort(hostUuid, inv.getUuid(), keyVersion, "vm-just-before-delete-from-db"); + } + + TpmDeletionMsg removeMsg = new TpmDeletionMsg(); + removeMsg.setVmInstanceUuid(inv.getUuid()); + removeMsg.setTpmUuid(tpmUuid); + removeMsg.setForceDelete(true); + bus.makeTargetServiceIdByResourceUuid(removeMsg, SERVICE_ID, removeMsg.getTpmUuid()); + MessageReply reply = bus.call(removeMsg); + if (!reply.isSuccess()) { + logger.warn(String.format("failed to remove TPM[uuid:%s] of VM[uuid:%s], error: %s", + tpmUuid, inv.getUuid(), reply.getError())); + } + } + + private void clearRollbackInfo(VmInstanceSpec spec) { + if (spec.getDevicesSpec() == null || spec.getDevicesSpec().getTpm() == null) { + return; + } + spec.getDevicesSpec().getTpm().setResourceKeyCreatedNew(false); + spec.getDevicesSpec().getTpm().setResourceKeyProviderUuid(null); + } + + private String findSourceTpmUuidFromSnapshotTpmBackupFile(String tpmBackupFileUuid) { + if (StringUtils.isBlank(tpmBackupFileUuid)) { + return null; + } + VmHostBackupFileVO bf = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.uuid, tpmBackupFileUuid) + .eq(VmHostBackupFileVO_.type, VmHostFileType.TpmState) + .find(); + if (bf == null || StringUtils.isBlank(bf.getResourceUuid())) { + return null; + } + String sourceVmUuid = Q.New(VolumeSnapshotGroupVO.class) + .select(VolumeSnapshotGroupVO_.vmInstanceUuid) + .eq(VolumeSnapshotGroupVO_.uuid, bf.getResourceUuid()) + .findValue(); + if (StringUtils.isBlank(sourceVmUuid)) { + return null; + } + return Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, sourceVmUuid) + .select(TpmVO_.uuid) + .findValue(); + } + + @Override + public void beforeMigrateVm(VmInstanceInventory inv, String destHostUuid) { + } + + @Override + public void preMigrateVm(VmInstanceInventory inv, String destHostUuid, Completion completion) { + if (inv == null || StringUtils.isBlank(destHostUuid)) { + completion.success(); + return; + } + String srcHostUuid = inv.getHostUuid(); + if (StringUtils.isBlank(srcHostUuid)) { + completion.success(); + return; + } + VtpmMigratePreAgentContext ctx = new VtpmMigratePreAgentContext(inv.getUuid(), srcHostUuid, destHostUuid); + ctx.setEnableKeyProvider(!VmGlobalConfig.ALLOWED_TPM_VM_WITHOUT_KMS.value(Boolean.class)); + String tpmUuid = VmTpmManager.findTpmUuidForVmOrNull(ctx.getVmUuid()); + if (StringUtils.isBlank(tpmUuid)) { + completion.success(); + return; + } + if (!ctx.isEnableKeyProvider()) { + completion.success(); + return; + } + try { + ctx.setTpmUuid(tpmUuid); + ensureVtpmKeyBindingAndDekLoaded(ctx); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + resolveSourceHostVtpmSecretUuid(ctx, new ReturnValueCompletion(completion) { + @Override + public void success(String sourceSecretUuid) { + ctx.setSourceSecretUuid(sourceSecretUuid); + ensureDestinationHostVtpmSecretDefined(ctx, new Completion(completion) { + @Override + public void success() { + completion.success(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void ensureVtpmKeyBindingAndDekLoaded(VtpmMigratePreAgentContext ctx) { + String tpmUuid = ctx.getTpmUuid(); + ctx.setProviderUuid(resourceKeyBackend.findKeyProviderUuidByTpm(tpmUuid)); + ctx.setProviderName(resourceKeyBackend.findKeyProviderNameByTpm(tpmUuid)); + if (StringUtils.isBlank(ctx.getProviderUuid()) && StringUtils.isBlank(ctx.getProviderName())) { + throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10163, "missing TPM resource key binding for tpm[uuid:%s] before migrate", tpmUuid)); + } + ctx.setKeyVersion(resourceKeyBackend.findKeyVersionByTpm(tpmUuid)); + if (ctx.getKeyVersion() == null) { + throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10163, "cannot find keyVersion for tpm[uuid:%s] before migrate", tpmUuid)); + } + + GetOrCreateResourceKeyContext keyCtx = new GetOrCreateResourceKeyContext(); + keyCtx.setResourceUuid(tpmUuid); + keyCtx.setResourceType(TpmVO.class.getSimpleName()); + keyCtx.setKeyProviderUuid(ctx.getProviderUuid()); + keyCtx.setKeyProviderName(ctx.getProviderName()); + keyCtx.setPurpose("vtpm"); + ResourceKeyResult result = resourceKeyManager.getKey(keyCtx); + if (StringUtils.isBlank(result.getDekBase64())) { + throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10163, "missing DEK for tpm[uuid:%s] after getKey before migrate", tpmUuid)); + } + ctx.setResourceKeyResult(result); + } + + private void resolveSourceHostVtpmSecretUuid(VtpmMigratePreAgentContext ctx, ReturnValueCompletion completion) { + ResolveVtpmLibvirtSecretOnHypervisorMsg m = new ResolveVtpmLibvirtSecretOnHypervisorMsg(); + m.setHostUuid(ctx.getSrcHostUuid()); + m.setVmUuid(ctx.getVmUuid()); + bus.makeTargetServiceIdByResourceUuid(m, HostConstant.SERVICE_ID, m.getHostUuid()); + bus.send(m, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + logger.warn(String.format("vTPM resolve libvirt secret uuid on agent failed, vmUuid:%s, err:%s", + ctx.getVmUuid(), r.getError().getReadableDetails())); + completion.fail(operr( + "cannot continue vTPM migrate precheck: failed to resolve libvirt secret UUID from domain XML, vmUuid:%s, srcHostUuid:%s", + ctx.getVmUuid(), ctx.getSrcHostUuid())); + return; + } + ResolveVtpmLibvirtSecretOnHypervisorReply rr = r.castReply(); + String sourceSecretUuid = StringUtils.isNotBlank(rr.getSecretUuid()) ? rr.getSecretUuid() : null; + if (sourceSecretUuid == null) { + completion.fail(operr( + "cannot continue vTPM migrate precheck: failed to resolve libvirt secret UUID from domain XML, vmUuid:%s, srcHostUuid:%s", + ctx.getVmUuid(), ctx.getSrcHostUuid())); + return; + } + logger.info(String.format( + "vTPM preMigrate source secret uuid resolved from domain XML, vmUuid:%s, srcHostUuid:%s, xmlHint:%s", + ctx.getVmUuid(), ctx.getSrcHostUuid(), sourceSecretUuid)); + completion.success(sourceSecretUuid); + } + }); + } + + private void ensureDestinationHostVtpmSecretDefined(VtpmMigratePreAgentContext ctx, Completion completion) { + ResourceKeyResult keyResult = ctx.getResourceKeyResult(); + if (keyResult == null || StringUtils.isBlank(keyResult.getDekBase64())) { + completion.fail(operr(ORG_ZSTACK_KVM_10163, "missing DEK for tpm[uuid:%s] before ensure secret on destination", ctx.getTpmUuid())); + return; + } + + SecretHostDefineMsg defMsg = new SecretHostDefineMsg(); + defMsg.setHostUuid(ctx.getDstHostUuid()); + defMsg.setVmUuid(ctx.getVmUuid()); + defMsg.setDekBase64(keyResult.getDekBase64()); + defMsg.setPurpose("vtpm"); + defMsg.setKeyVersion(ctx.getKeyVersion()); + defMsg.setUsageInstance(HOST_SECRET_USAGE_INSTANCE_VTPM); + defMsg.setSecretUuid(ctx.getSourceSecretUuid()); + defMsg.setDescription(String.format("Define secret for VM %s before live migration", ctx.getVmUuid())); + bus.makeTargetServiceIdByResourceUuid(defMsg, HostConstant.SERVICE_ID, defMsg.getHostUuid()); + bus.send(defMsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply defReply) { + if (!defReply.isSuccess()) { + completion.fail(defReply.getError()); + return; + } + + SecretHostDefineReply defR = defReply.castReply(); + if (StringUtils.isBlank(defR.getSecretUuid())) { + completion.fail(operr( + "define secret on host succeeded but returned empty secretUuid, hostUuid:%s", ctx.getDstHostUuid())); + return; + } + String destSecretUuid = defR.getSecretUuid(); + logger.info(String.format( + "vTPM preMigrate destination secret defined, vmUuid:%s, dstHostUuid:%s, secretUuid:%s", + ctx.getVmUuid(), ctx.getDstHostUuid(), destSecretUuid)); + completion.success(); + } + }); + } + + @Override + public void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid, NoErrorCompletion completion) { + String vmUuid = inv == null ? null : inv.getUuid(); + String destHostUuid = inv == null ? null : inv.getHostUuid(); + if (StringUtils.isBlank(vmUuid) || StringUtils.isBlank(srcHostUuid) || srcHostUuid.equals(destHostUuid)) { + completion.done(); + return; + } + + if (!isVmCurrentlyOnExpectedHost(vmUuid, destHostUuid)) { + completion.done(); + return; + } + + Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid); + deleteHostSecretBestEffort(srcHostUuid, vmUuid, keyVersion, "after-migrate"); + completion.done(); + } + + @Override + public void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason, NoErrorCompletion completion) { + String vmUuid = inv == null ? null : inv.getUuid(); + if (StringUtils.isBlank(vmUuid) || StringUtils.isBlank(destHostUuid)) { + completion.done(); + return; + } + + Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid); + deleteHostSecretBestEffort(destHostUuid, vmUuid, keyVersion, "failed-migrate"); + completion.done(); + } + + @Override + public void vmStateChanged(VmInstanceInventory vm, VmInstanceState oldState, VmInstanceState newState) { + String vmUuid = vm == null ? null : vm.getUuid(); + if (StringUtils.isBlank(vmUuid)) { + logger.info(String.format("vmStateChanged skip: vmUuid is blank, oldState=%s, newState=%s", oldState, newState)); + return; + } + + // Record source host when storage migration starts. In some end-state events (e.g. VolumeMigrating->Stopped), + // inventory host fields may not carry both src/dst host values reliably. + if (newState == VmInstanceState.VolumeMigrating) { + String srcHostUuid = vm.getLastHostUuid(); + if (StringUtils.isBlank(srcHostUuid)) { + srcHostUuid = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.lastHostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findValue(); + } + if (StringUtils.isNotBlank(srcHostUuid)) { + volumeMigratingSourceHostCache.put(vmUuid, srcHostUuid); + logger.info(String.format( + "vmStateChanged cache volume-migrating src host: vm[uuid:%s], oldState=%s, newState=%s, srcHostUuid=%s", + vmUuid, oldState, newState, srcHostUuid)); + } else { + logger.info(String.format( + "vmStateChanged skip cache: source host is blank, vm[uuid:%s], oldState=%s, newState=%s", + vmUuid, oldState, newState)); + } + return; + } + + if (oldState != VmInstanceState.VolumeMigrating) { + return; + } + + String srcHostUuid = volumeMigratingSourceHostCache.remove(vmUuid); + if (StringUtils.isBlank(srcHostUuid)) { + logger.info(String.format( + "vmStateChanged skip delete: no cached source host, vm[uuid:%s], oldState=%s, newState=%s", + vmUuid, oldState, newState)); + return; + } + + String destHostUuid = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findValue(); + if (StringUtils.isBlank(destHostUuid)) { + destHostUuid = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.lastHostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findValue(); + } + if (StringUtils.isBlank(destHostUuid) || srcHostUuid.equals(destHostUuid)) { + logger.info(String.format( + "vmStateChanged skip delete: invalid host mapping, vm[uuid:%s], oldState=%s, newState=%s, srcHostUuid=%s, destHostUuid=%s", + vmUuid, oldState, newState, srcHostUuid, destHostUuid)); + return; + } + + Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid); + logger.info(String.format( + "vmStateChanged trigger delete: vm[uuid:%s], srcHostUuid=%s, destHostUuid=%s, keyVersion=%s, reason=volume-migrated-host-change", + vmUuid, srcHostUuid, destHostUuid, keyVersion)); + deleteHostSecretBestEffort(srcHostUuid, vmUuid, keyVersion, "volume-migrated-host-change"); + } + + @Override + public void vmAfterExpunge(VmInstanceInventory vm) { + String vmUuid = vm.getUuid(); + Integer keyVersion = findTpmKeyVersionByVmUuid(vmUuid); + + java.util.Set hostUuids = new java.util.HashSet<>(); + if (StringUtils.isNotBlank(vm.getHostUuid())) { + hostUuids.add(vm.getHostUuid()); + } + if (StringUtils.isNotBlank(vm.getLastHostUuid())) { + hostUuids.add(vm.getLastHostUuid()); + } + + if (hostUuids.isEmpty()) { + return; + } + + for (String hostUuid : hostUuids) { + deleteHostSecretBestEffort(hostUuid, vmUuid, keyVersion, "expunge"); + } + } + + private Integer findTpmKeyVersionByVmUuid(String vmUuid) { + if (StringUtils.isBlank(vmUuid)) { + return null; + } + String tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .select(TpmVO_.uuid) + .findValue(); + return tpmUuid == null ? null : resourceKeyBackend.findKeyVersionByTpm(tpmUuid); + } + + private boolean isVmCurrentlyOnExpectedHost(String vmUuid, String expectedHostUuid) { + if (StringUtils.isBlank(vmUuid) || StringUtils.isBlank(expectedHostUuid)) { + return false; + } + + String currentHostUuid = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.hostUuid) + .eq(VmInstanceVO_.uuid, vmUuid) + .findValue(); + return expectedHostUuid.equals(currentHostUuid); + } + + private static boolean isVtpmSecretNotFoundOnHost(ErrorCode errorCode) { + if (SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND.equals(errorCode.getCode())) { + return true; + } + String details = errorCode.getDetails(); + return details != null && details.contains(SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND); + } + + private void deleteHostSecretBestEffort(String hostUuid, String vmUuid, Integer keyVersion, String reason) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(vmUuid) || keyVersion == null) { + logger.info(String.format( + "skip delete host secret: reason=%s, hostUuid=%s, vmUuid=%s, keyVersion=%s", + reason, hostUuid, vmUuid, keyVersion)); + return; + } + + logger.info(String.format( + "send delete host secret: reason=%s, hostUuid=%s, vmUuid=%s, keyVersion=%s", + reason, hostUuid, vmUuid, keyVersion)); + SecretHostDeleteMsg dmsg = new SecretHostDeleteMsg(); + dmsg.setHostUuid(hostUuid); + dmsg.setVmUuid(vmUuid); + dmsg.setPurpose("vtpm"); + dmsg.setKeyVersion(keyVersion); + dmsg.setUsageInstance(HOST_SECRET_USAGE_INSTANCE_VTPM); + bus.makeTargetServiceIdByResourceUuid(dmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(dmsg, new CloudBusCallBack(null) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + ErrorCode err = reply.getError(); + String errMsg = err != null && err.getDetails() != null ? err.getDetails() : "unknown error"; + logger.warn(String.format( + "best-effort delete host secret failed on %s for vm[uuid:%s], host[uuid:%s]: %s", + reason, vmUuid, hostUuid, errMsg)); + } + } + }); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmManager.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmManager.java new file mode 100644 index 00000000000..0ab602cef63 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmManager.java @@ -0,0 +1,1085 @@ +package org.zstack.kvm.tpm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.VmGlobalConfig; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend; +import org.zstack.compute.vm.devices.VmTpmManager; +import org.zstack.core.Platform; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.MessageSafe; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.core.db.SQLBatch; +import org.zstack.core.thread.ChainTask; +import org.zstack.core.thread.SyncTaskChain; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.core.timeout.TimeHelper; +import org.zstack.core.workflow.SimpleFlowChain; +import org.zstack.header.AbstractService; +import org.zstack.header.core.Completion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostConstant; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.Message; +import org.zstack.header.message.MessageReply; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.secret.SecretHostDeleteMsg; +import org.zstack.header.tpm.api.APIAddTpmEvent; +import org.zstack.header.tpm.api.APIAddTpmMsg; +import org.zstack.header.tpm.api.APIGetTpmCapabilityMsg; +import org.zstack.header.tpm.api.APIGetTpmCapabilityReply; +import org.zstack.header.tpm.api.APIRemoveTpmEvent; +import org.zstack.header.tpm.api.APIRemoveTpmMsg; +import org.zstack.header.tpm.api.APIUpdateTpmMsg; +import org.zstack.header.tpm.entity.TpmCapabilityView; +import org.zstack.header.tpm.entity.TpmInventory; +import org.zstack.header.tpm.entity.TpmKeyBackupVO; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.tpm.message.AddTpmMsg; +import org.zstack.header.tpm.message.AddTpmReply; +import org.zstack.header.tpm.message.DeleteTpmKeyBackupMsg; +import org.zstack.header.tpm.message.DeleteTpmKeyBackupReply; +import org.zstack.header.tpm.message.RestoreTpmEncryptionKeyMsg; +import org.zstack.header.tpm.message.RestoreTpmEncryptionKeyReply; +import org.zstack.header.tpm.message.TpmDeletionMsg; +import org.zstack.header.tpm.message.TpmDeletionReply; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.additions.ResetVmTpmMsg; +import org.zstack.header.vm.additions.ResetVmTpmReply; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostFileInventory; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.header.vm.additions.VmHostFileOperation; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.kvm.KVMConstant; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KvmCommandSender; +import org.zstack.kvm.KvmResponseWrapper; +import org.zstack.kvm.efi.KvmSecureBootExtensions; +import org.zstack.header.tpm.message.BackupTpmEncryptionKeyMsg; +import org.zstack.header.tpm.message.BackupTpmEncryptionKeyReply; +import org.zstack.kvm.tpm.message.CloneVmTpmMsg; +import org.zstack.kvm.tpm.message.CloneVmTpmReply; +import org.zstack.resourceconfig.ResourceConfig; +import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.utils.CollectionUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.zstack.compute.vm.VmGlobalConfig.RESET_TPM_AFTER_VM_CLONE; +import static org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend.*; +import static org.zstack.core.Platform.err; +import static org.zstack.core.Platform.operr; +import static org.zstack.header.errorcode.SysErrors.NOT_SUPPORTED; +import static org.zstack.header.tpm.TpmConstants.*; +import static org.zstack.header.tpm.TpmErrors.VM_STATE_ERROR; +import static org.zstack.kvm.KVMSystemTags.EDK_RPM_TOKEN; +import static org.zstack.kvm.KVMSystemTags.SWTPM_VERSION; +import static org.zstack.kvm.KVMSystemTags.SWTPM_VERSION_TOKEN; +import static org.zstack.kvm.KVMSystemTags.VM_EDK; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.CollectionUtils.isEmpty; +import static org.zstack.utils.CollectionUtils.transform; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class KvmTpmManager extends AbstractService { + private static final CLogger logger = Utils.getLogger(KvmTpmManager.class); + + @Autowired + private CloudBus bus; + @Autowired + private ThreadFacade threadFacade; + @Autowired + private ResourceConfigFacade resourceConfigFacade; + @Autowired + private VmTpmManager vmTpmManager; + @Autowired + private TpmEncryptedResourceKeyBackend tpmKeyBackend; + @Autowired + private EncryptedResourceKeyManager resourceKeyManager; + @Autowired + private KvmSecureBootExtensions secureBootExtensions; + @Autowired + private DatabaseFacade databaseFacade; + @Autowired + private TimeHelper timeHelper; + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } + + @Override + public String getId() { + return bus.makeLocalServiceId(SERVICE_ID); + } + + private String tpmQueueSyncSignature(String vmUuid) { + return String.format("tpm-queue-sync-%s", vmUuid); + } + + @MessageSafe + public void handleMessage(Message msg) { + if (msg instanceof APIMessage) { + handleApiMessage((APIMessage) msg); + } else { + handleLocalMessage(msg); + } + } + + private void handleLocalMessage(Message msg) { + if (msg instanceof AddTpmMsg) { + handle((AddTpmMsg) msg); + } else if (msg instanceof TpmDeletionMsg) { + handle((TpmDeletionMsg) msg); + } else if (msg instanceof CloneVmTpmMsg) { + handle((CloneVmTpmMsg) msg); + } else if (msg instanceof BackupTpmEncryptionKeyMsg) { + handle((BackupTpmEncryptionKeyMsg) msg); + } else if (msg instanceof RestoreTpmEncryptionKeyMsg) { + handle((RestoreTpmEncryptionKeyMsg) msg); + } else if (msg instanceof DeleteTpmKeyBackupMsg) { + handle((DeleteTpmKeyBackupMsg) msg); + } else if (msg instanceof ResetVmTpmMsg) { + handle((ResetVmTpmMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handleApiMessage(APIMessage msg) { + if (msg instanceof APIGetTpmCapabilityMsg) { + handle((APIGetTpmCapabilityMsg) msg); + } else if (msg instanceof APIAddTpmMsg) { + handle((APIAddTpmMsg) msg); + } else if (msg instanceof APIRemoveTpmMsg) { + handle((APIRemoveTpmMsg) msg); + } else if (msg instanceof APIUpdateTpmMsg) { + handle((APIUpdateTpmMsg) msg); + } else { + bus.dealWithUnknownMessage(msg); + } + } + + private void handle(AddTpmMsg msg) { + if (msg.getKeyProviderUuid() != null && msg.getResourceUuidKeyFrom() != null) { + throw new OperationFailureException(operr(ORG_ZSTACK_KVM_10163, + "keyProviderUuid and resourceUuidKeyFrom cannot be set at the same time")); + } + + AddTpmReply reply = new AddTpmReply(); + threadFacade.chainSubmit(new ChainTask(msg) { + @Override + public void run(SyncTaskChain chain) { + AddTpmToVmContext context = AddTpmToVmContext.valueOf(msg); + addTpmToVm(context, new Completion(chain, msg) { + @Override + public void success() { + chain.next(); + TpmVO vo = Q.New(TpmVO.class) + .eq(TpmVO_.uuid, msg.getTpmUuid()) + .find(); + reply.setInventory(TpmInventory.valueOf(vo)); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + chain.next(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + @Override + public String getSyncSignature() { + return tpmQueueSyncSignature(msg.getVmInstanceUuid()); + } + + @Override + public String getName() { + return "queue-of-add-tpm-to-vm-" + msg.getVmInstanceUuid(); + } + }); + } + + static class AddTpmToVmContext { + String keyProviderUuid; // create new key with the provider uuid + String resourceUuidKeyFrom; // copy key from the resource uuid + String vmInstanceUuid; + String tpmUuid; + + boolean tpmCreated; + boolean keyProviderAttached; + String createdTpmUuid; + + static AddTpmToVmContext valueOf(AddTpmMsg msg) { + AddTpmToVmContext context = new AddTpmToVmContext(); + context.keyProviderUuid = msg.getKeyProviderUuid(); + context.resourceUuidKeyFrom = msg.getResourceUuidKeyFrom(); + context.vmInstanceUuid = msg.getVmInstanceUuid(); + context.tpmUuid = msg.getTpmUuid(); + return context; + } + } + + private void addTpmToVm(AddTpmToVmContext context, Completion completion) { + SimpleFlowChain.of("add-tpm-to-vm-" + context.vmInstanceUuid) + .then(Flow.of("check-vm-status") + .handle(trigger -> { + VmInstanceVO vm = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, context.vmInstanceUuid) + .find(); + + if (!SUPPORT_VM_STATES_FOR_TPM_OPERATION.contains(vm.getState())) { + trigger.fail(err(ORG_ZSTACK_KVM_10163, VM_STATE_ERROR, + "The current VM state does not support adding TPM operations") + .withOpaque("support.vm.state", SUPPORT_VM_STATES_FOR_TPM_OPERATION)); + return; + } + trigger.next(); + }) + .build()) + .then(Flow.of("create-tpm-db-records") + .handle(trigger -> { + TpmVO tpm = vmTpmManager.persistTpmVO(context.tpmUuid, context.vmInstanceUuid); + context.createdTpmUuid = tpm.getUuid(); + context.tpmCreated = true; + trigger.next(); + }) + .rollback(trigger -> { + if (context.tpmCreated && context.createdTpmUuid != null) { + vmTpmManager.deleteTpmVO(context.createdTpmUuid); + } + trigger.rollback(); + }) + .build()) + .then(Flow.of("attach-key-provider-to-tpm") + .skipIf(data -> VmGlobalConfig.ALLOWED_TPM_VM_WITHOUT_KMS.value(Boolean.class) || context.keyProviderUuid == null) + .handle(trigger -> { + tpmKeyBackend.attachKeyProviderToTpm(context.createdTpmUuid, context.keyProviderUuid); + context.keyProviderAttached = true; + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext keyCtx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + keyCtx.setResourceUuid(context.createdTpmUuid); + keyCtx.setResourceType(TpmVO.class.getSimpleName()); + keyCtx.setKeyProviderUuid(context.keyProviderUuid); + keyCtx.setPurpose("vtpm"); + resourceKeyManager.getOrCreateKey(keyCtx, new ReturnValueCompletion(trigger) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult returnValue) { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + }) + .rollback(trigger -> { + if (context.keyProviderAttached && context.createdTpmUuid != null) { + tpmKeyBackend.detachKeyProviderFromTpm(context.createdTpmUuid); + } + trigger.rollback(); + }) + .build()) + .then(Flow.of("copy-key-to-tpm") + .skipIf(data -> VmGlobalConfig.ALLOWED_TPM_VM_WITHOUT_KMS.value(Boolean.class) || context.resourceUuidKeyFrom == null) + .handle(trigger -> { + RestoreEncryptedResourceKeyContext restoreContext = new RestoreEncryptedResourceKeyContext(); + restoreContext.srcResourceUuid = context.resourceUuidKeyFrom; + restoreContext.dstResourceUuid = context.createdTpmUuid; + tpmKeyBackend.restoreEncryptedResourceKey(restoreContext); + trigger.next(); + }) + .rollback(trigger -> { + if (context.createdTpmUuid != null) { + tpmKeyBackend.detachKeyProviderFromTpm(context.createdTpmUuid); + } + trigger.rollback(); + }) + .build()) + .propagateExceptionTo(completion) + .done(completion::success) + .error(completion::fail) + .start(); + } + + private void handle(TpmDeletionMsg msg) { + TpmDeletionReply reply = new TpmDeletionReply(); + threadFacade.chainSubmit(new ChainTask(msg) { + @Override + public void run(SyncTaskChain chain) { + RemoveTpmFromVmContext context = RemoveTpmFromVmContext.valueOf(msg); + removeTpmFromVm(context, new Completion(chain, msg) { + @Override + public void success() { + chain.next(); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + chain.next(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + @Override + public String getSyncSignature() { + return tpmQueueSyncSignature(msg.getVmInstanceUuid()); + } + + @Override + public String getName() { + return "queue-of-remove-tpm-from-vm-" + msg.getVmInstanceUuid(); + } + }); + } + + static class RemoveTpmFromVmContext { + String vmInstanceUuid; + String tpmUuid; + Integer keyVersion; + + // enable when TPM delete/VM delete operation + boolean force; + + List hostFiles; + + static RemoveTpmFromVmContext valueOf(TpmDeletionMsg msg) { + RemoveTpmFromVmContext context = new RemoveTpmFromVmContext(); + context.vmInstanceUuid = msg.getVmInstanceUuid(); + context.tpmUuid = msg.getTpmUuid(); + context.force = msg.isForceDelete(); + return context; + } + } + + private void removeTpmFromVm(RemoveTpmFromVmContext context, Completion completion) { + SimpleFlowChain.of("remove-tpm-from-vm-" + context.vmInstanceUuid) + .then(Flow.of("check-vm-status") + .skipIf(data -> context.force) + .handle(trigger -> { + VmInstanceVO vm = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, context.vmInstanceUuid) + .find(); + + if (!SUPPORT_VM_STATES_FOR_TPM_OPERATION.contains(vm.getState())) { + trigger.fail(err(ORG_ZSTACK_KVM_10163, VM_STATE_ERROR, + "The current VM state does not support removing TPM operations") + .withOpaque("support.vm.state", SUPPORT_VM_STATES_FOR_TPM_OPERATION)); + return; + } + trigger.next(); + }) + .build()) + .then(Flow.of("collect-vm-host-files") + .handle(trigger -> { + context.keyVersion = tpmKeyBackend.findKeyVersionByTpm(context.tpmUuid); + // DO NOT delete NvRam type VmHostFile: Maybe secure boot or other component related. + context.hostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, context.vmInstanceUuid) + .eq(VmHostFileVO_.type, VmHostFileType.TpmState) + .list(); + trigger.next(); + }) + .build()) + .then(Flow.of("send-delete-commands-to-hosts") + .skipIf(data -> context.hostFiles.isEmpty()) + .handle(trigger -> { + Map> filesByHost = new HashMap<>(); + for (VmHostFileVO file : context.hostFiles) { + filesByHost.computeIfAbsent(file.getHostUuid(), k -> new ArrayList<>()).add(file); + } + + new While<>(filesByHost.entrySet()).each((entry, whileCompletion) -> { + String hostUuid = entry.getKey(); + List files = entry.getValue(); + + KVMAgentCommands.WriteVmHostFileContentCmd cmd = new KVMAgentCommands.WriteVmHostFileContentCmd(); + List fileTOs = new ArrayList<>(); + for (VmHostFileVO file : files) { + KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); + to.setPath(file.getPath()); + to.setType(file.getType().toString()); + to.setOperation(VmHostFileOperation.Delete.toString()); + fileTOs.add(to); + } + cmd.setHostFiles(fileTOs); + + new KvmCommandSender(hostUuid).send(cmd, KVMConstant.WRITE_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.WriteVmHostFileContentResponse rsp = + wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class); + return rsp.isSuccess() ? null : operr(ORG_ZSTACK_KVM_10163, "failed to delete host files on host[uuid=%s]", hostUuid); + }, new ReturnValueCompletion(whileCompletion) { + @Override + public void success(KvmResponseWrapper wrapper) { + whileCompletion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("failed to delete host files on host[uuid=%s], but continuing with DB cleanup: %s", + hostUuid, errorCode.getDetails())); + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + trigger.next(); + } + }); + }) + .build()) + .then(Flow.of("delete-host-secret") + .skipIf(data -> context.keyVersion == null) + .handle(trigger -> { + Set hostUuids = new HashSet<>(); + for (VmHostFileVO file : context.hostFiles) { + hostUuids.add(file.getHostUuid()); + } + if (hostUuids.isEmpty()) { + addVmCurrentAndLastHostUuidsForSecretDelete(hostUuids, context.vmInstanceUuid); + } + if (hostUuids.isEmpty()) { + trigger.next(); + return; + } + + new While<>(new ArrayList<>(hostUuids)).each((hostUuid, whileCompletion) -> { + SecretHostDeleteMsg dmsg = new SecretHostDeleteMsg(); + dmsg.setHostUuid(hostUuid); + dmsg.setVmUuid(context.vmInstanceUuid); + dmsg.setPurpose("vtpm"); + dmsg.setKeyVersion(context.keyVersion); + dmsg.setUsageInstance(KVMConstant.HOST_SECRET_USAGE_INSTANCE_VTPM); + bus.makeTargetServiceIdByResourceUuid(dmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(dmsg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + ErrorCode err = reply.getError(); + String errMsg = err != null && err.getDetails() != null ? err.getDetails() : "unknown error"; + logger.warn(String.format("failed to delete host secret on host[uuid:%s] for vm[uuid:%s], continue cleanup: %s", + hostUuid, context.vmInstanceUuid, errMsg)); + } + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + trigger.next(); + } + }); + }) + .build()) + .then(Flow.of("detach-resource-key") + .handle(trigger -> { + tpmKeyBackend.detachKeyProviderFromTpm(context.tpmUuid); + List backupUuidList = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, context.vmInstanceUuid) + .eq(VmHostBackupFileVO_.type, VmHostFileType.TpmState) + .select(VmHostBackupFileVO_.uuid) + .listValues(); + for (String backupUuid : backupUuidList) { + tpmKeyBackend.cleanEncryptedResourceKey(backupUuid); + } + trigger.next(); + }) + .build()) + .then(Flow.of("remove-db-records") + .handle(trigger -> { + new SQLBatch() { + @Override + protected void scripts() { + sql(TpmVO.class) + .eq(TpmVO_.uuid, context.tpmUuid) + .delete(); + sql(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, context.vmInstanceUuid) + .eq(VmHostFileVO_.type, VmHostFileType.TpmState) + .delete(); + sql(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, context.vmInstanceUuid) + .eq(VmHostBackupFileVO_.type, VmHostFileType.TpmState) + .delete(); + } + }.execute(); + trigger.next(); + }) + .build()) + .propagateExceptionTo(completion) + .done(completion::success) + .error(completion::fail) + .start(); + } + + private void handle(CloneVmTpmMsg msg) { + CloneVmTpmReply reply = new CloneVmTpmReply(); + + String originTpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, msg.getSrcVmUuid()) + .select(TpmVO_.uuid) + .findValue(); + if (originTpmUuid == null) { + bus.reply(msg, reply); + return; + } + + SimpleFlowChain.of("clone-VM-TPM") + .then(Flow.of("persist-TPM-VO") + .handle(trigger -> { + reply.setInventories(new ArrayList<>()); + for (String dstVmUuid : msg.getDstVmUuidList()) { + TpmVO dstTpm = vmTpmManager.persistTpmVO(null, dstVmUuid); + reply.getInventories().add(TpmInventory.valueOf(dstTpm)); + } + trigger.next(); + }) + .rollback(trigger -> { + if (CollectionUtils.isEmpty(reply.getInventories())) { + trigger.rollback(); + return; + } + + new While<>(reply.getInventories()).each((tpm, whileCompletion) -> { + RemoveTpmFromVmContext removeContext = new RemoveTpmFromVmContext(); + removeContext.vmInstanceUuid = tpm.getVmInstanceUuid(); + removeContext.tpmUuid = tpm.getUuid(); + removeContext.force = true; + removeTpmFromVm(removeContext, new Completion(whileCompletion) { + @Override + public void success() { + whileCompletion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("failed to delete tpm for VM[%s] but still continue: %s", + tpm.getVmInstanceUuid(), errorCode.getReadableDetails())); + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + trigger.rollback(); + } + }); + }) + .build()) + .then(Flow.of("clone-encrypted-resource-key-if-needed") + .handle(trigger -> { + boolean resetTpm; + if (msg.getResetTpm() == null) { + ResourceConfig resourceConfig = resourceConfigFacade.getResourceConfig(RESET_TPM_AFTER_VM_CLONE.getIdentity()); + resetTpm = resourceConfig.getResourceConfigValue(msg.getSrcVmUuid(), Boolean.class); + } else { + resetTpm = msg.getResetTpm(); + } + + new While<>(reply.getInventories()).each((inventory, whileCompletion) -> { + TpmEncryptedResourceKeyBackend.CloneEncryptedResourceKeyContext context = + new TpmEncryptedResourceKeyBackend.CloneEncryptedResourceKeyContext(); + context.srcTpmUuid = originTpmUuid; + context.dstTpmUuid = inventory.getUuid(); + context.resetTpm = resetTpm; + tpmKeyBackend.cloneEncryptedResourceKey(context, new Completion(whileCompletion) { + @Override + public void success() { + whileCompletion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + whileCompletion.addError(errorCode); + whileCompletion.allDone(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (errorCodeList.getCauses().isEmpty()) { + trigger.next(); + return; + } + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "Failed to clone encrypted resource key") + .withOpaque("src.tpm.uuid", originTpmUuid) + .causedBy(errorCodeList)); + } + }); + }) + .build()) + .propagateExceptionTo(msg) + .done(() -> bus.reply(msg, reply)) + .error(errorCode -> { + reply.setError(errorCode); + bus.reply(msg, reply); + }) + .start(); + } + + private void handle(BackupTpmEncryptionKeyMsg msg) { + BackupEncryptedResourceKeyContext content = new BackupEncryptedResourceKeyContext(); + content.srcResourceUuid = msg.getSrcResourceUuid(); + content.dstResourceUuid = msg.getDstResourceUuid(); + tpmKeyBackend.backupEncryptedResourceKey(content); + bus.reply(msg, new BackupTpmEncryptionKeyReply()); + } + + private void handle(RestoreTpmEncryptionKeyMsg msg) { + RestoreTpmEncryptionKeyReply reply = new RestoreTpmEncryptionKeyReply(); + String tpmKeyBackupUuid = null; + try { + if (msg.isBackupCurrentKey() + && msg.getDstResourceUuid() != null + && tpmKeyBackend.checkTpmKeyProviderAttached(msg.getDstResourceUuid())) { + tpmKeyBackupUuid = Platform.getUuid(); + TpmKeyBackupVO backupVo = new TpmKeyBackupVO(); + backupVo.setUuid(tpmKeyBackupUuid); + Timestamp now = new Timestamp(timeHelper.getCurrentTimeMillis()); + backupVo.setCreateDate(now); + backupVo.setLastOpDate(now); + databaseFacade.persist(backupVo); + BackupEncryptedResourceKeyContext backupCtx = new BackupEncryptedResourceKeyContext(); + backupCtx.srcResourceUuid = msg.getDstResourceUuid(); + backupCtx.dstResourceUuid = tpmKeyBackupUuid; + tpmKeyBackend.backupEncryptedResourceKey(backupCtx); + reply.setTpmKeyBackupUuid(tpmKeyBackupUuid); + } + + RestoreEncryptedResourceKeyContext content = new RestoreEncryptedResourceKeyContext(); + content.srcResourceUuid = msg.getSrcResourceUuid(); + content.dstResourceUuid = msg.getDstResourceUuid(); + tpmKeyBackend.restoreEncryptedResourceKey(content); + bus.reply(msg, reply); + } catch (Exception t) { + if (tpmKeyBackupUuid != null) { + tpmKeyBackend.cleanTpmKeyBackupEncryptedResourceKey(tpmKeyBackupUuid); + databaseFacade.removeByPrimaryKey(tpmKeyBackupUuid, TpmKeyBackupVO.class); + } + throw t; + } + } + + private void handle(DeleteTpmKeyBackupMsg msg) { + DeleteTpmKeyBackupReply reply = new DeleteTpmKeyBackupReply(); + if (msg.getTpmKeyBackupUuid() != null) { + tpmKeyBackend.cleanTpmKeyBackupEncryptedResourceKey(msg.getTpmKeyBackupUuid()); + databaseFacade.removeByPrimaryKey(msg.getTpmKeyBackupUuid(), TpmKeyBackupVO.class); + } + bus.reply(msg, reply); + } + + static class ResetVmTpmContext { + String vmInstanceUuid; + Integer keyVersion; + + List hostFiles; + VmHostFileVO hostFileToDeleteLast; + List hostFileUuidListDeleteSuccessfully = new ArrayList<>(); + ErrorCodeList errorsOnSendCmd = new ErrorCodeList(); + + static ResetVmTpmContext valueOf(ResetVmTpmMsg msg) { + ResetVmTpmContext context = new ResetVmTpmContext(); + context.vmInstanceUuid = msg.getVmInstanceUuid(); + return context; + } + } + + private void handle(ResetVmTpmMsg msg) { + ResetVmTpmReply reply = new ResetVmTpmReply(); + threadFacade.chainSubmit(new ChainTask(msg) { + @Override + public void run(SyncTaskChain chain) { + ResetVmTpmContext context = ResetVmTpmContext.valueOf(msg); + resetVmTpm(context, new Completion(chain, msg) { + @Override + public void success() { + chain.next(); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + chain.next(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + @Override + public String getSyncSignature() { + return tpmQueueSyncSignature(msg.getVmInstanceUuid()); + } + + @Override + public String getName() { + return "queue-of-reset-tpm-from-vm-" + msg.getVmInstanceUuid(); + } + }); + } + + private void resetVmTpm(ResetVmTpmContext context, Completion completion) { + String vmUuid = context.vmInstanceUuid; + String tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .select(TpmVO_.uuid) + .findValue(); + if (tpmUuid != null) { + context.keyVersion = tpmKeyBackend.findKeyVersionByTpm(tpmUuid); + } + + SimpleFlowChain.of("reset-vm-tpm-" + vmUuid) + .then(Flow.of("collect-vm-host-files") + .handle(trigger -> { + context.hostFiles = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.type, VmHostFileType.TpmState) + .orderByAsc(VmHostFileVO_.lastOpDate) + .list(); + if (!context.hostFiles.isEmpty()) { + // We should delete it in last turn: + context.hostFileToDeleteLast = context.hostFiles.get(context.hostFiles.size() - 1); + context.hostFiles.remove(context.hostFiles.size() - 1); + } + trigger.next(); + }) + .build()) + .then(Flow.of("send-delete-commands-to-hosts-exclude-last-modified") + .skipIf(data -> context.hostFiles.isEmpty()) + .handle(trigger -> { + Map> filesByHost = new HashMap<>(); + for (VmHostFileVO file : context.hostFiles) { + filesByHost.computeIfAbsent(file.getHostUuid(), k -> new ArrayList<>()).add(file); + } + + new While<>(filesByHost.entrySet()).each((entry, whileCompletion) -> { + List fileTOs = new ArrayList<>(); + for (VmHostFileVO file : entry.getValue()) { + KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); + to.setPath(file.getPath()); + to.setType(file.getType().toString()); + to.setOperation(VmHostFileOperation.Delete.toString()); + fileTOs.add(to); + } + + KvmSecureBootExtensions.RewriteVmHostFilesContext ctx = + new KvmSecureBootExtensions.RewriteVmHostFilesContext(); + ctx.hostUuid = entry.getKey(); + ctx.hostFiles = fileTOs; + + secureBootExtensions.rewriteVmHostFiles(ctx, new Completion(whileCompletion) { + @Override + public void success() { + context.hostFileUuidListDeleteSuccessfully.addAll( + transform(entry.getValue(), VmHostFileVO::getUuid)); + whileCompletion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + context.errorsOnSendCmd.getCauses().add(errorCode.withOpaque("host.uuid", entry.getKey())); + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + trigger.next(); + } + }); + }) + .build()) + .then(Flow.of("remove-db-records") + .skipIf(data -> context.hostFileUuidListDeleteSuccessfully.isEmpty()) + .handle(trigger -> { + SQL.New(VmHostFileVO.class) + .in(VmHostFileVO_.uuid, context.hostFileUuidListDeleteSuccessfully) + .delete(); + trigger.next(); + }) + .build()) + .then(Flow.of("check-if-any-error-in-command-sending") + .handle(trigger -> { + // If any host failed to delete, abort the chain to preserve + // the last-modified TPM record as a recovery point. + if (!context.errorsOnSendCmd.getCauses().isEmpty()) { + if (context.errorsOnSendCmd.getCauses().size() == 1) { + trigger.fail(context.errorsOnSendCmd.getCauses().get(0)); + } else { + trigger.fail(operr(ORG_ZSTACK_KVM_10163, "failed to delete TPM files on multiple hosts") + .withOpaque("vm.uuid", vmUuid) + .causedBy(context.errorsOnSendCmd.getCauses())); + } + return; + } + trigger.next(); + }) + .build()) + .then(Flow.of("send-delete-commands-to-hosts-for-last-modified") + .skipIf(data -> context.hostFileToDeleteLast == null) + .handle(trigger -> { + KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); + to.setPath(context.hostFileToDeleteLast.getPath()); + to.setType(context.hostFileToDeleteLast.getType().toString()); + to.setOperation(VmHostFileOperation.Delete.toString()); + + KvmSecureBootExtensions.RewriteVmHostFilesContext ctx = + new KvmSecureBootExtensions.RewriteVmHostFilesContext(); + ctx.hostUuid = context.hostFileToDeleteLast.getHostUuid(); + ctx.hostFiles = list(to); + + secureBootExtensions.rewriteVmHostFiles(ctx, new Completion(trigger) { + @Override + public void success() { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode.withOpaque("host.uuid", ctx.hostUuid)); + } + }); + }) + .build()) + .then(Flow.of("remove-host-file-db-records-for-remains") + .skipIf(data -> context.hostFileToDeleteLast == null) + .handle(trigger -> { + VmHostFileDeletionMsg deletionMsg = new VmHostFileDeletionMsg(); + deletionMsg.setUuid(context.hostFileToDeleteLast.getUuid()); + deletionMsg.setForceDelete(false); + bus.makeLocalServiceId(deletionMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(deletionMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + }) + .build()) + .then(Flow.of("remove-backups-db-records-for-remains") + .handle(trigger -> { + List backupUuidList = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, vmUuid) + .eq(VmHostBackupFileVO_.type, VmHostFileType.TpmState) + .select(VmHostBackupFileVO_.uuid) + .listValues(); + if (isEmpty(backupUuidList)) { + trigger.next(); + return; + } + + new While<>(backupUuidList).each((uuid, whileCompletion) -> { + VmHostBackupFileDeletionMsg deletionMsg = new VmHostBackupFileDeletionMsg(); + deletionMsg.setUuid(uuid); + // VmHostFileVO has been deleted in the previous step, so force delete here is safe + deletionMsg.setForceDelete(true); + bus.makeLocalServiceId(deletionMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(deletionMsg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + whileCompletion.addError(reply.getError()); + } + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + String details = String.join("\n", transform(errorCodeList.getCauses(), ErrorCode::getReadableDetails)); + logger.warn("failed to clean backup files but still continue:\n" + details); + } + trigger.next(); + } + }); + }) + .build()) + .then(Flow.of("delete-host-secret") + .handle(trigger -> { + if (context.keyVersion == null) { + trigger.next(); + return; + } + Set hostUuids = new HashSet<>(); + for (VmHostFileVO file : context.hostFiles) { + hostUuids.add(file.getHostUuid()); + } + if (context.hostFileToDeleteLast != null) { + hostUuids.add(context.hostFileToDeleteLast.getHostUuid()); + } + if (hostUuids.isEmpty()) { + addVmCurrentAndLastHostUuidsForSecretDelete(hostUuids, vmUuid); + } + if (hostUuids.isEmpty()) { + trigger.next(); + return; + } + + new While<>(new ArrayList<>(hostUuids)).each((hostUuid, whileCompletion) -> { + SecretHostDeleteMsg dmsg = new SecretHostDeleteMsg(); + dmsg.setHostUuid(hostUuid); + dmsg.setVmUuid(vmUuid); + dmsg.setPurpose("vtpm"); + dmsg.setKeyVersion(context.keyVersion); + dmsg.setUsageInstance(KVMConstant.HOST_SECRET_USAGE_INSTANCE_VTPM); + bus.makeTargetServiceIdByResourceUuid(dmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(dmsg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + ErrorCode err = reply.getError(); + String errMsg = err != null && err.getDetails() != null ? err.getDetails() : "unknown error"; + logger.warn(String.format("failed to delete host secret on host[uuid:%s] for vm[uuid:%s], continue reset: %s", + hostUuid, vmUuid, errMsg)); + } + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + trigger.next(); + } + }); + }) + .build()) + .propagateExceptionTo(completion) + .done(completion::success) + .error(completion::fail) + .start(); + } + + private static void addVmCurrentAndLastHostUuidsForSecretDelete(Set hostUuids, String vmInstanceUuid) { + if (vmInstanceUuid == null) { + return; + } + VmInstanceVO vm = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmInstanceUuid) + .find(); + if (vm == null) { + return; + } + if (vm.getHostUuid() != null) { + hostUuids.add(vm.getHostUuid()); + } + if (vm.getLastHostUuid() != null) { + hostUuids.add(vm.getLastHostUuid()); + } + } + + private void handle(APIGetTpmCapabilityMsg msg) { + TpmCapabilityView view = new TpmCapabilityView(); + + final TpmVO tpm = Q.New(TpmVO.class) + .eq(TpmVO_.uuid, msg.getTpmUuid()) + .find(); + final VmInstanceVO vm = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, tpm.getVmInstanceUuid()) + .find(); + view.setTpmInventory(TpmInventory.valueOf(tpm)); + + List files = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vm.getUuid()) + .in(VmHostFileVO_.type, list(VmHostFileType.TpmState, VmHostFileType.NvRam)) + .list(); + view.setFileRefs(VmHostFileInventory.valueOf(files)); + + view.setEdkVersion(VM_EDK.getTokenByResourceUuid(vm.getUuid(), EDK_RPM_TOKEN)); + + if (vm.getHostUuid() != null) { + view.setSwtpmVersion(SWTPM_VERSION.getTokenByResourceUuid(vm.getHostUuid(), SWTPM_VERSION_TOKEN)); + } else if (vm.getLastHostUuid() != null) { + view.setSwtpmVersion(SWTPM_VERSION.getTokenByResourceUuid(vm.getLastHostUuid(), SWTPM_VERSION_TOKEN)); + } + + ResourceConfig resourceConfig = resourceConfigFacade.getResourceConfig(RESET_TPM_AFTER_VM_CLONE.getIdentity()); + view.setResetTpmAfterVmCloneConfig(resourceConfig.getResourceConfigValue(vm.getUuid(), Boolean.class)); + + APIGetTpmCapabilityReply reply = new APIGetTpmCapabilityReply(); + reply.setInventory(view); + bus.reply(msg, reply); + } + + private void handle(APIAddTpmMsg msg) { + APIAddTpmEvent event = new APIAddTpmEvent(msg.getId()); + + AddTpmMsg inner = AddTpmMsg.valueOf(msg); + bus.makeTargetServiceIdByResourceUuid(inner, SERVICE_ID, msg.getResourceUuid()); + bus.send(inner, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + event.setInventory(((AddTpmReply) reply.castReply()).getInventory()); + } else { + event.setError(reply.getError()); + } + bus.publish(event); + } + }); + } + + private void handle(APIRemoveTpmMsg msg) { + APIRemoveTpmEvent event = new APIRemoveTpmEvent(msg.getId()); + + TpmDeletionMsg inner = TpmDeletionMsg.valueOf(msg); + bus.makeTargetServiceIdByResourceUuid(inner, SERVICE_ID, msg.getTpmUuid()); + bus.send(inner, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + event.setError(reply.getError()); + } + bus.publish(event); + } + }); + } + + private void handle(APIUpdateTpmMsg msg) { + throw new OperationFailureException(err(ORG_ZSTACK_KVM_10163, NOT_SUPPORTED, + "UpdateTpm is not supported in current version")); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/SnapshotGroupRevertTpmHelper.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/SnapshotGroupRevertTpmHelper.java new file mode 100644 index 00000000000..39d5045dd76 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/SnapshotGroupRevertTpmHelper.java @@ -0,0 +1,138 @@ +package org.zstack.kvm.tpm; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend; +import org.zstack.core.db.Q; +import org.zstack.header.tpm.entity.TpmSpec; +import org.zstack.header.vm.APICreateVmInstanceFromVolumeSnapshotGroupMsg; +import org.zstack.header.vm.CreateVmInstanceMsg; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.devices.NvRamSpec; +import org.zstack.header.vm.devices.VmDevicesSpec; +import org.zstack.kvm.KVMSystemTags; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +import static org.zstack.compute.vm.VmGlobalConfig.ALLOWED_TPM_VM_WITHOUT_KMS; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class SnapshotGroupRevertTpmHelper { + private static final CLogger logger = Utils.getLogger(SnapshotGroupRevertTpmHelper.class); + + @Autowired + private TpmEncryptedResourceKeyBackend tpmKeyBackend; + + public void setupFromApi(APICreateVmInstanceFromVolumeSnapshotGroupMsg apiMsg, CreateVmInstanceMsg cmsg) { + String snapshotGroupUuid = apiMsg.getVolumeSnapshotGroupUuid(); + + boolean resetTpm = apiMsg.getResetTpm() == Boolean.TRUE; + List backupFiles = Q.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.resourceUuid, snapshotGroupUuid) + .list(); + + if (backupFiles.isEmpty()) { + logger.debug(String.format( + "no VmHostBackupFileVO found for volume snapshot group[uuid:%s], skip restoring TPM/NvRam", + snapshotGroupUuid)); + return; + } + + VmHostBackupFileVO tpmBackupFile = null; + VmHostBackupFileVO nvRamBackupFile = null; + for (VmHostBackupFileVO f : backupFiles) { + if (f.getType() == VmHostFileType.TpmState) { + tpmBackupFile = f; + } else if (f.getType() == VmHostFileType.NvRam) { + nvRamBackupFile = f; + } + } + + if (tpmBackupFile == null && nvRamBackupFile == null) { + logger.debug(String.format("no TpmState or NvRam backup file found for volume snapshot group[uuid:%s]", + snapshotGroupUuid)); + return; + } + + VmDevicesSpec devicesSpec = cmsg.getDevicesSpec(); + if (devicesSpec == null) { + devicesSpec = new VmDevicesSpec(); + cmsg.setDevicesSpec(devicesSpec); + } + + if (tpmBackupFile != null) { + TpmSpec tpmSpec = devicesSpec.getTpm(); + if (tpmSpec == null) { + tpmSpec = new TpmSpec(); + devicesSpec.setTpm(tpmSpec); + } + setupTpmSpec(snapshotGroupUuid, tpmBackupFile, tpmSpec, cmsg, resetTpm); + } + + if (nvRamBackupFile != null) { + NvRamSpec nvRamSpec = devicesSpec.getNvRam(); + if (nvRamSpec == null) { + nvRamSpec = new NvRamSpec(); + devicesSpec.setNvRam(nvRamSpec); + } + nvRamSpec.setBackupFileUuid(nvRamBackupFile.getUuid()); + logger.debug(String.format("set NvRam restore info for volume snapshot group[uuid:%s], nvRamBackupFileUuid:%s", + snapshotGroupUuid, nvRamBackupFile.getUuid())); + } + } + + private void setupTpmSpec(String snapshotGroupUuid, VmHostBackupFileVO tpmBackupFile, TpmSpec tpmSpec, + CreateVmInstanceMsg cmsg, boolean resetTpm) { + tpmSpec.setEnable(true); + + if (resetTpm) { + // resetTpm=true: reset generate a new one during VM creation + logger.debug(String.format("resetTpm is true for volume snapshot group[uuid:%s], " + + "will reset tpmBackupFileUuid:%s", snapshotGroupUuid, tpmBackupFile.getUuid())); + } else { + tpmSpec.setBackupFileUuid(tpmBackupFile.getUuid()); + } + + if (ALLOWED_TPM_VM_WITHOUT_KMS.value(Boolean.class) == Boolean.TRUE) { + return; + } + + if (resetTpm) { + String defaultProviderUuid = tpmKeyBackend.defaultKeyProviderUuid(); + tpmSpec.setKeyProviderUuid(defaultProviderUuid); + logger.info(String.format( + "snapshot-reset TPM target provider selected, source[snapshotGroupUuid:%s,tpmBackupFileUuid:%s], " + + "destination[vmResourceUuid:%s,vmName:%s,providerUuid:%s]", + snapshotGroupUuid, tpmBackupFile.getUuid(), + cmsg.getResourceUuid(), cmsg.getName(), defaultProviderUuid)); + return; + } + + String keyProviderName = KVMSystemTags.TPM_KEY_PROVIDER_NAME + .getTokenByResourceUuid(tpmBackupFile.getUuid(), KVMSystemTags.TPM_KEY_PROVIDER_NAME_TOKEN); + if (keyProviderName == null) { + logger.warn(String.format( + "failed to find keyProvider from snapshotGroup[uuid:%s] by tpmBackupFile[uuid:%s]", + snapshotGroupUuid, tpmBackupFile.getUuid())); + if (tpmSpec.getKeyProviderUuid() == null) { + tpmSpec.setKeyProviderUuid(tpmKeyBackend.defaultKeyProviderUuid()); + } + return; + } + + String keyProviderUuid = tpmKeyBackend.findKeyProviderUuidByName(keyProviderName); + if (keyProviderUuid == null) { + logger.warn(String.format( + "failed to resolve keyProvider[name:%s] from snapshotGroup[uuid:%s] by tpmBackupFile[uuid:%s], keep keyProviderUuid unset", + keyProviderName, snapshotGroupUuid, tpmBackupFile.getUuid())); + return; + } + + tpmSpec.setKeyProviderUuid(keyProviderUuid); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmStateVmHostBackupFileBase.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmStateVmHostBackupFileBase.java new file mode 100644 index 00000000000..65be7cce42e --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmStateVmHostBackupFileBase.java @@ -0,0 +1,95 @@ +package org.zstack.kvm.tpm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend; +import org.zstack.core.db.Q; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.kvm.KVMSystemTags; +import org.zstack.kvm.vmfiles.AbstractVmHostBackupFileBase; +import org.zstack.tag.SystemTagCreator; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import static org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend.BackupEncryptedResourceKeyContext; +import static org.zstack.utils.CollectionDSL.e; +import static org.zstack.utils.CollectionDSL.map; + +public class TpmStateVmHostBackupFileBase extends AbstractVmHostBackupFileBase { + private static final CLogger logger = Utils.getLogger(TpmStateVmHostBackupFileBase.class); + + @Autowired + private TpmEncryptedResourceKeyBackend resourceKeyBackend; + + public TpmStateVmHostBackupFileBase(VmHostBackupFileVO self) { + super(self); + } + + @Override + public VmHostFileType type() { + return VmHostFileType.TpmState; + } + + @Override + public void afterBackup(VmHostBackupFileVO from) { + String keyProviderName = KVMSystemTags.TPM_KEY_PROVIDER_NAME + .getTokenByResourceUuid(from.getUuid(), KVMSystemTags.TPM_KEY_PROVIDER_NAME_TOKEN); + if (keyProviderName == null) { + logger.debug(String.format("no tpm key provider name system tag found on source VmHostBackupFileVO[uuid:%s], skip copying", + from.getUuid())); + return; + } + + createKeyProviderNameTag(keyProviderName); + logger.debug(String.format("copied tpm key provider name[%s] from VmHostBackupFileVO[uuid:%s] to VmHostBackupFileVO[uuid:%s]", + keyProviderName, from.getUuid(), self.getUuid())); + } + + @Override + public void afterBackup(VmHostFileVO from) { + String tpmUuid = Q.New(TpmVO.class) + .select(TpmVO_.uuid) + .eq(TpmVO_.vmInstanceUuid, from.getVmInstanceUuid()) + .findValue(); + if (tpmUuid == null) { + logger.debug(String.format("no TpmVO found for vm[uuid:%s], skip creating key provider name tag on VmHostBackupFileVO[uuid:%s]", + from.getVmInstanceUuid(), self.getUuid())); + return; + } + + String keyProviderName = resourceKeyBackend.findKeyProviderNameByTpm(tpmUuid); + if (keyProviderName == null) { + logger.debug(String.format("no key provider name found for tpm[uuid:%s] of vm[uuid:%s], skip creating tag on VmHostBackupFileVO[uuid:%s]", + tpmUuid, from.getVmInstanceUuid(), self.getUuid())); + return; + } + + createKeyProviderNameTag(keyProviderName); + logger.debug(String.format("created tpm key provider name[%s] tag on VmHostBackupFileVO[uuid:%s] from tpm[uuid:%s] of vm[uuid:%s]", + keyProviderName, self.getUuid(), tpmUuid, from.getVmInstanceUuid())); + + final BackupEncryptedResourceKeyContext context = new BackupEncryptedResourceKeyContext(); + context.srcResourceUuid = tpmUuid; + context.dstResourceUuid = self.getUuid(); + resourceKeyBackend.backupEncryptedResourceKey(context); + } + + @SuppressWarnings("unchecked") + private void createKeyProviderNameTag(String keyProviderName) { + SystemTagCreator creator = KVMSystemTags.TPM_KEY_PROVIDER_NAME.newSystemTagCreator(self.getUuid()); + creator.setTagByTokens(map(e(KVMSystemTags.TPM_KEY_PROVIDER_NAME_TOKEN, keyProviderName))); + creator.inherent = true; + creator.recreate = true; + creator.create(); + } + + @Override + public void clean() { + logger.debug(String.format("clean TpmState VmBackupFileVO[uuid=%s]", self.getUuid())); + resourceKeyBackend.cleanEncryptedResourceKey(self.getUuid()); + super.clean(); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmStateVmHostFileBase.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmStateVmHostFileBase.java new file mode 100644 index 00000000000..32f00975a41 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmStateVmHostFileBase.java @@ -0,0 +1,16 @@ +package org.zstack.kvm.tpm; + +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.kvm.vmfiles.AbstractVmHostFileBase; + +public class TpmStateVmHostFileBase extends AbstractVmHostFileBase { + public TpmStateVmHostFileBase(VmHostFileVO self) { + super(self); + } + + @Override + public VmHostFileType type() { + return VmHostFileType.TpmState; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmTO.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmTO.java new file mode 100644 index 00000000000..073257172f2 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/TpmTO.java @@ -0,0 +1,33 @@ +package org.zstack.kvm.tpm; + +import java.io.Serializable; + +public class TpmTO implements Serializable { + private String keyProviderUuid; + private String secretUuid; + private String installPath; + + public String getKeyProviderUuid() { + return keyProviderUuid; + } + + public void setKeyProviderUuid(String keyProviderUuid) { + this.keyProviderUuid = keyProviderUuid; + } + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/VtpmMigratePreAgentContext.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/VtpmMigratePreAgentContext.java new file mode 100644 index 00000000000..06d7b7c6784 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/VtpmMigratePreAgentContext.java @@ -0,0 +1,102 @@ +package org.zstack.kvm.tpm; + +import org.zstack.header.keyprovider.EncryptedResourceKeyManager.ResourceKeyResult; + +import java.util.Objects; + +/** + * Context for vTPM secret preparation on the source KVM host immediately before the agent + * {@code migrate-vm} call. + *

+ * Built during {@link org.zstack.header.vm.VmInstanceMigrateExtensionPoint#preMigrateVm} (e.g. in + * {@link KvmTpmExtensions}) to hold VM/source/destination hosts and resolved TPM/key/secret fields. + * This replaces the previous split between a thin migrate wrapper + * and a separate object meant for FlowChain {@code Map} storage. + */ +public final class VtpmMigratePreAgentContext { + private final String vmUuid; + private final String srcHostUuid; + private final String dstHostUuid; + private boolean enableKeyProvider = true; + + private String tpmUuid; + private String providerUuid; + private String providerName; + private Integer keyVersion; + private ResourceKeyResult resourceKeyResult; + private String sourceSecretUuid; + + public VtpmMigratePreAgentContext(String vmUuid, String srcHostUuid, String dstHostUuid) { + this.vmUuid = Objects.requireNonNull(vmUuid, "vmUuid"); + this.srcHostUuid = Objects.requireNonNull(srcHostUuid, "srcHostUuid"); + this.dstHostUuid = Objects.requireNonNull(dstHostUuid, "dstHostUuid"); + } + + public String getVmUuid() { + return vmUuid; + } + + public String getSrcHostUuid() { + return srcHostUuid; + } + + public String getDstHostUuid() { + return dstHostUuid; + } + + public boolean isEnableKeyProvider() { + return enableKeyProvider; + } + + public void setEnableKeyProvider(boolean enableKeyProvider) { + this.enableKeyProvider = enableKeyProvider; + } + + public String getTpmUuid() { + return tpmUuid; + } + + public void setTpmUuid(String tpmUuid) { + this.tpmUuid = tpmUuid; + } + + public String getProviderUuid() { + return providerUuid; + } + + public void setProviderUuid(String providerUuid) { + this.providerUuid = providerUuid; + } + + public String getProviderName() { + return providerName; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public Integer getKeyVersion() { + return keyVersion; + } + + public void setKeyVersion(Integer keyVersion) { + this.keyVersion = keyVersion; + } + + public ResourceKeyResult getResourceKeyResult() { + return resourceKeyResult; + } + + public void setResourceKeyResult(ResourceKeyResult resourceKeyResult) { + this.resourceKeyResult = resourceKeyResult; + } + + public String getSourceSecretUuid() { + return sourceSecretUuid; + } + + public void setSourceSecretUuid(String sourceSecretUuid) { + this.sourceSecretUuid = sourceSecretUuid; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/message/CloneVmTpmMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/message/CloneVmTpmMsg.java new file mode 100644 index 00000000000..60722841ed2 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/message/CloneVmTpmMsg.java @@ -0,0 +1,35 @@ +package org.zstack.kvm.tpm.message; + +import org.zstack.header.message.NeedReplyMessage; + +import java.util.List; + +public class CloneVmTpmMsg extends NeedReplyMessage { + private String srcVmUuid; + private List dstVmUuidList; + private Boolean resetTpm; + + public String getSrcVmUuid() { + return srcVmUuid; + } + + public void setSrcVmUuid(String srcVmUuid) { + this.srcVmUuid = srcVmUuid; + } + + public List getDstVmUuidList() { + return dstVmUuidList; + } + + public void setDstVmUuidList(List dstVmUuidList) { + this.dstVmUuidList = dstVmUuidList; + } + + public Boolean getResetTpm() { + return resetTpm; + } + + public void setResetTpm(Boolean resetTpm) { + this.resetTpm = resetTpm; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/message/CloneVmTpmReply.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/message/CloneVmTpmReply.java new file mode 100644 index 00000000000..0c9be5fb4f3 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/message/CloneVmTpmReply.java @@ -0,0 +1,18 @@ +package org.zstack.kvm.tpm.message; + +import org.zstack.header.message.MessageReply; +import org.zstack.header.tpm.entity.TpmInventory; + +import java.util.List; + +public class CloneVmTpmReply extends MessageReply { + private List inventories; + + public List getInventories() { + return inventories; + } + + public void setInventories(List inventories) { + this.inventories = inventories; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/AbstractVmHostBackupFileBase.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/AbstractVmHostBackupFileBase.java new file mode 100644 index 00000000000..36b14ce9ffe --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/AbstractVmHostBackupFileBase.java @@ -0,0 +1,34 @@ +package org.zstack.kvm.vmfiles; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.SQL; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; + +import java.util.Objects; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public abstract class AbstractVmHostBackupFileBase { + protected VmHostBackupFileVO self; + + protected AbstractVmHostBackupFileBase(VmHostBackupFileVO self) { + this.self = Objects.requireNonNull(self); + } + + public abstract VmHostFileType type(); + + public void afterBackup(VmHostBackupFileVO from) { + } + + public void afterBackup(VmHostFileVO from) { + } + + public void clean() { + SQL.New(VmHostBackupFileVO.class) + .eq(VmHostBackupFileVO_.uuid, self.getUuid()) + .delete(); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/AbstractVmHostFileBase.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/AbstractVmHostFileBase.java new file mode 100644 index 00000000000..8e87dcf70cd --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/AbstractVmHostFileBase.java @@ -0,0 +1,19 @@ +package org.zstack.kvm.vmfiles; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; + +import java.util.Objects; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public abstract class AbstractVmHostFileBase { + protected VmHostFileVO self; + + protected AbstractVmHostFileBase(VmHostFileVO self) { + this.self = Objects.requireNonNull(self); + } + + public abstract VmHostFileType type(); +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileAllocatorExtensionPoint.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileAllocatorExtensionPoint.java new file mode 100644 index 00000000000..15cf08f1f70 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileAllocatorExtensionPoint.java @@ -0,0 +1,66 @@ +package org.zstack.kvm.vmfiles; + +import org.zstack.header.allocator.HostAllocatorFilterExtensionPoint; +import org.zstack.header.allocator.HostAllocatorSpec; +import org.zstack.header.host.HostVO; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; +import org.zstack.core.db.Q; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.zstack.core.Platform.i18n; + +public class VmHostFileAllocatorExtensionPoint implements HostAllocatorFilterExtensionPoint { + private static final CLogger logger = Utils.getLogger(VmHostFileAllocatorExtensionPoint.class); + + @Override + public List filterHostCandidates(List candidates, HostAllocatorSpec spec) { + if (spec == null) { + return candidates; + } + + String vmOperation = spec.getVmOperation(); + if (vmOperation == null || !VmInstanceConstant.VmOperation.Start.toString().equals(vmOperation)) { + return candidates; + } + + if (spec.getVmInstance() == null) { + return candidates; + } + + String vmUuid = spec.getVmInstance().getUuid(); + String lastHostUuid = spec.getVmInstance().getLastHostUuid(); + + List files = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, lastHostUuid) + .select(VmHostFileVO_.type) + .notNull(VmHostFileVO_.changeDate) + .listValues(); + if (files.isEmpty()) { + return candidates; + } + + String types = files.stream() + .map(Enum::name) + .collect(Collectors.joining(",")); + + logger.debug(String.format("only allow VM[uuid:%s] start on last host[uuid:%s], unsynchronized %s VM host files exist", + vmUuid, lastHostUuid, types)); + return candidates.stream() + .filter(c -> Objects.equals(lastHostUuid, c.getUuid())) + .collect(Collectors.toList()); + } + + @Override + public String filterErrorReason() { + return i18n("only allowed start on last host: unsynchronized VM host files exist"); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileAutoCompleter.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileAutoCompleter.java new file mode 100644 index 00000000000..6dccf665bb1 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileAutoCompleter.java @@ -0,0 +1,69 @@ +package org.zstack.kvm.vmfiles; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.compute.vm.VmGlobalConfig; +import org.zstack.header.apimediator.ApiMessageInterceptionException; +import org.zstack.header.apimediator.GlobalApiMessageInterceptor; +import org.zstack.header.message.APIMessage; +import org.zstack.core.db.Q; +import org.zstack.header.vm.APICreateVmInstanceFromVolumeSnapshotGroupMsg; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_; +import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +import static org.zstack.utils.CollectionDSL.list; + +public class VmHostFileAutoCompleter implements GlobalApiMessageInterceptor { + private static final CLogger logger = Utils.getLogger(VmHostFileAutoCompleter.class); + + @Autowired + private ResourceConfigFacade resourceConfigFacade; + + @Override + @SuppressWarnings("rawtypes") + public List getMessageClassToIntercept() { + return list( + APICreateVmInstanceFromVolumeSnapshotGroupMsg.class + ); + } + + @Override + public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionException { + if (msg instanceof APICreateVmInstanceFromVolumeSnapshotGroupMsg) { + validate((APICreateVmInstanceFromVolumeSnapshotGroupMsg) msg); + } + return msg; + } + + @Override + public InterceptorPosition getPosition() { + return InterceptorPosition.FRONT; + } + + private void validate(APICreateVmInstanceFromVolumeSnapshotGroupMsg msg) { + if (msg.getResetTpm() != null) { + return; + } + + String snapshotGroupUuid = msg.getVolumeSnapshotGroupUuid(); + if (snapshotGroupUuid == null) { + return; + } + + String vmUuid = Q.New(VolumeSnapshotGroupVO.class) + .select(VolumeSnapshotGroupVO_.vmInstanceUuid) + .eq(VolumeSnapshotGroupVO_.uuid, snapshotGroupUuid) + .findValue(); + if (vmUuid == null) { + return; + } + + Boolean resolved = resourceConfigFacade.getResourceConfigValue( + VmGlobalConfig.RESET_TPM_AFTER_VM_CLONE, vmUuid, Boolean.class); + msg.setResetTpm(resolved); + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileTracker.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileTracker.java new file mode 100644 index 00000000000..e5caf259e54 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/VmHostFileTracker.java @@ -0,0 +1,567 @@ +package org.zstack.kvm.vmfiles; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.asyncbatch.While; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.EventCallback; +import org.zstack.core.cloudbus.EventFacade; +import org.zstack.core.cloudbus.ResourceDestinationMaker; +import org.zstack.core.config.GlobalConfigUpdateExtensionPoint; +import org.zstack.core.db.Q; +import org.zstack.core.db.SQL; +import org.zstack.core.thread.PeriodicTask; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.core.timeout.TimeHelper; +import org.zstack.header.Component; +import org.zstack.header.core.FutureReturnValueCompletion; +import org.zstack.header.core.WhileDoneCompletion; +import org.zstack.header.errorcode.ErrorCodeList; +import org.zstack.header.host.HostVO; +import org.zstack.header.host.HostVO_; +import org.zstack.header.message.MessageReply; +import org.zstack.header.vm.VmCanonicalEvents; +import org.zstack.header.vm.VmInstanceAO_; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceEO; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostFileContentVO; +import org.zstack.header.vm.additions.VmHostFileContentVO_; +import org.zstack.header.vm.additions.VmHostFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostFileOperation; +import org.zstack.header.vm.additions.VmHostFileSyncReason; +import org.zstack.header.vm.additions.VmHostFileType; +import org.zstack.header.vm.additions.VmHostFileVO; +import org.zstack.header.vm.additions.VmHostFileVO_; +import org.zstack.header.vo.ResourceVO; +import org.zstack.header.vo.ResourceVO_; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMConstant; +import org.zstack.kvm.KVMGlobalConfig; +import org.zstack.kvm.KvmCommandSender; +import org.zstack.kvm.vmfiles.message.SyncVmHostFilesFromHostMsg; +import org.zstack.utils.CollectionUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.data.Pair; +import org.zstack.utils.logging.CLogger; + +import javax.persistence.Tuple; +import java.sql.Timestamp; +import java.util.*; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.zstack.core.Platform.operr; +import static org.zstack.utils.CollectionUtils.transformToSet; +import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; + +public class VmHostFileTracker implements Component { + private static final CLogger logger = Utils.getLogger(VmHostFileTracker.class); + + private static final long CLEANUP_INTERVAL_SECONDS = 1800; + private static final long ONE_DAY_MS = TimeUnit.DAYS.toMillis(1); + private static final long SEVEN_DAYS_MS = TimeUnit.DAYS.toMillis(7); + private static final long NINETY_DAYS_MS = TimeUnit.DAYS.toMillis(90); + private static final long KVM_CMD_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(3); + + @Autowired + private ThreadFacade threadFacade; + @Autowired + private CloudBus bus; + @Autowired + private ResourceDestinationMaker destinationMaker; + @Autowired + private TimeHelper timeHelper; + @Autowired + private EventFacade eventFacade; + + private Future trackerThread; + private Future cleanupThread; + + @Override + public boolean start() { + if (CoreGlobalProperty.UNIT_TEST_ON) { + logger.info("VmHostFileTracker is disabled in unit test"); + return true; + } + + submitTrackerTask(); + submitCleanupTask(); + setupCanonicalEvents(); + + GlobalConfigUpdateExtensionPoint listener = (oldConfig, newConfig) -> { + logger.debug(String.format("%s changed from %s to %s, restarting vm-host-file-tracker", + newConfig.getCanonicalName(), oldConfig.value(), newConfig.value())); + submitTrackerTask(); + }; + + KVMGlobalConfig.VM_HOST_FILE_SYNC_INTERVAL.installUpdateExtension(listener); + KVMGlobalConfig.VM_HOST_FILE_SYNC_CONCURRENCY.installUpdateExtension(listener); + + return true; + } + + @Override + public boolean stop() { + if (trackerThread != null) { + trackerThread.cancel(true); + } + if (cleanupThread != null) { + cleanupThread.cancel(true); + } + return true; + } + + private void setupCanonicalEvents() { + eventFacade.on(VmCanonicalEvents.VM_HOST_FILE_CHANGED_PATH, new EventCallback() { + @Override + protected void run(Map tokens, Object data) { + if (!(data instanceof VmCanonicalEvents.VmHostFileChangedData)) { + return; + } + + VmCanonicalEvents.VmHostFileChangedData d = (VmCanonicalEvents.VmHostFileChangedData) data; + String hostUuid = d.getHostUuid(); + String vmUuid = d.getVmUuid(); + List types = d.getTypes(); + + if (hostUuid == null || vmUuid == null || types == null || types.isEmpty()) { + logger.warn(String.format("received VM_HOST_FILE_CHANGED event with incomplete data: hostUuid=%s, vmUuid=%s", hostUuid, vmUuid)); + return; + } + + Timestamp now = new Timestamp(timeHelper.getCurrentTimeMillis()); + for (String type : types) { + VmHostFileType fileType; + try { + fileType = VmHostFileType.valueOf(type); + } catch (Exception e) { + logger.warn(String.format("ignore invalid vm host file type in event: vmUuid=%s, hostUuid=%s, type=%s", + vmUuid, hostUuid, type), e); + continue; + } + + long updated = SQL.New(VmHostFileVO.class) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.type, fileType) + .set(VmHostFileVO_.changeDate, now) + .update(); + + if (updated > 0) { + logger.debug(String.format("marked VmHostFile changed: vmUuid=%s, hostUuid=%s, type=%s", vmUuid, hostUuid, type)); + } else { + logger.warn(String.format("no VmHostFileVO found for vmUuid=%s, hostUuid=%s, type=%s", vmUuid, hostUuid, type)); + } + } + } + }); + } + + private synchronized void submitTrackerTask() { + if (trackerThread != null) { + trackerThread.cancel(true); + } + + long interval = KVMGlobalConfig.VM_HOST_FILE_SYNC_INTERVAL.value(Long.class); + trackerThread = threadFacade.submitPeriodicTask(new PeriodicTask() { + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + public long getInterval() { + return interval; + } + + @Override + public String getName() { + return "vm-host-file-tracker"; + } + + @Override + public void run() { + logger.info("VmHostFileTracker: starting periodic check of VM host files"); + syncVmHostFiles(); + } + }); + } + + private synchronized void submitCleanupTask() { + if (cleanupThread != null) { + cleanupThread.cancel(true); + } + + cleanupThread = threadFacade.submitPeriodicTask(new PeriodicTask() { + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + public long getInterval() { + return CLEANUP_INTERVAL_SECONDS; + } + + @Override + public String getName() { + return "vm-host-file-cleanup-tracker"; + } + + @Override + public void run() { + logger.info("VmHostFileTracker: starting periodic cleanup of expired VM host files"); + try { + cleanupExpiredVmHostFiles(); + } catch (Throwable t) { + logger.warn("error during VmHostFile cleanup", t); + } + try { + cleanupExpiredVmHostBackupFiles(); + } catch (Throwable t) { + logger.warn("error during VmHostBackupFile cleanup", t); + } + } + }); + } + + /** + * Vm host files the periodic sync should consider: (1) running VM files on the current host, + * and (2) stopped VM files on the last host when at least one is dirty ({@code changeDate} set). + */ + private List listVmHostFilesToSync() { + List hostFiles = new ArrayList<>(Q.New(VmHostFileVO.class, VmInstanceVO.class) + .table0() + .eq(VmHostFileVO_.vmInstanceUuid).table1(VmInstanceAO_.uuid) + .eq(VmHostFileVO_.hostUuid).table1(VmInstanceAO_.hostUuid) + .selectThisTable() + .table1() + .eq(VmInstanceAO_.state, VmInstanceState.Running) + .list()); + hostFiles.addAll(Q.New(VmHostFileVO.class, VmInstanceVO.class) + .table0() + .eq(VmHostFileVO_.vmInstanceUuid).table1(VmInstanceAO_.uuid) + .eq(VmHostFileVO_.hostUuid).table1(VmInstanceAO_.lastHostUuid) + .notNull(VmHostFileVO_.changeDate) + .selectThisTable() + .table1() + .eq(VmInstanceAO_.state, VmInstanceState.Stopped) + .notNull(VmInstanceAO_.lastHostUuid) + .list()); + return hostFiles; + } + + private void syncVmHostFiles() { + List hostFiles = listVmHostFilesToSync(); + if (hostFiles.isEmpty()) { + return; + } + + Map> grouped = hostFiles.stream() + .collect(Collectors.groupingBy(f -> f.getVmInstanceUuid() + "::" + f.getHostUuid())); + List> groups = new ArrayList<>(grouped.values()); + + long now = timeHelper.getCurrentTimeMillis(); + long checkIntervalMs = KVMGlobalConfig.VM_HOST_FILE_SYNC_INTERVAL.value(Long.class) * 1000; + long forceSyncThresholdMs = checkIntervalMs * 100; + int concurrency = KVMGlobalConfig.VM_HOST_FILE_SYNC_CONCURRENCY.value(Integer.class); + + new While<>(groups).step((group, whileCompletion) -> { + VmHostFileVO first = group.get(0); + String vmUuid = first.getVmInstanceUuid(); + String hostUuid = first.getHostUuid(); + + if (!destinationMaker.isManagedByUs(vmUuid)) { + whileCompletion.done(); + return; + } + + boolean hasChanged = group.stream().anyMatch(f -> f.getChangeDate() != null); + String syncReason; + if (hasChanged) { + syncReason = VmHostFileSyncReason.PeriodicDirtyCheck.reason(); + } else { + // check if force sync is needed based on lastSyncDate + Timestamp oldestLastSync = group.stream() + .map(VmHostFileVO::getLastSyncDate) + .filter(Objects::nonNull) + .min(Comparator.naturalOrder()) + .orElse(null); + + if (oldestLastSync != null && (now - oldestLastSync.getTime()) < forceSyncThresholdMs) { + whileCompletion.done(); + return; + } + syncReason = VmHostFileSyncReason.PeriodicForceSync.reason(); + } + + SyncVmHostFilesFromHostMsg syncMsg = new SyncVmHostFilesFromHostMsg(); + syncMsg.setHostUuid(hostUuid); + syncMsg.setVmUuid(vmUuid); + syncMsg.setSyncReason(syncReason); + + for (VmHostFileVO file : group) { + if (file.getType() == VmHostFileType.NvRam) { + syncMsg.setNvRamPath(file.getPath()); + } else if (file.getType() == VmHostFileType.TpmState) { + syncMsg.setTpmStateFolder(file.getPath()); + } + } + + bus.makeLocalServiceId(syncMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(syncMsg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.warn(String.format("failed to sync VM host file[vmUuid=%s] from host[uuid=%s]: %s", + vmUuid, hostUuid, reply.getError().getReadableDetails())); + } + whileCompletion.done(); + } + }); + }, concurrency).run(new WhileDoneCompletion(null) { + @Override + public void done(ErrorCodeList errorCodeList) { + // periodic check round finished, empty callback + } + }); + } + + /** + * active VM: In each type, max(lastOpDate) will not be deleted; + * delete lastOpDate > 1d if delete command success + * VmInstanceEO(deleted): delete lastOpDate > 90d if record is latest; + * delete lastOpDate > 7d if record is not latest even if delete command failed + * delete lastOpDate < 7d if record is not latest and delete command success + * VmInstanceEO(not exists): delete + * Host not exists: delete and not send command + */ + private void cleanupExpiredVmHostFiles() { + List vmTypeList = Q.New(VmHostFileVO.class) + .select(VmHostFileVO_.vmInstanceUuid, VmHostFileVO_.type) + .listTuple(); + if (vmTypeList.isEmpty()) { + return; + } + + Set> pairs = new HashSet<>(); + for (Tuple tuple : vmTypeList) { + pairs.add(new Pair<>(tuple.get(0, String.class), tuple.get(1, VmHostFileType.class))); + } + + long now = timeHelper.getCurrentTimeMillis(); + CollectionUtils.safeForEach(pairs, pair -> { + String vmUuid = pair.first(); + if (!destinationMaker.isManagedByUs(vmUuid)) { + return; + } + + VmHostFileType type = pair.second(); + List files = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.type, type) + .list(); + if (files.isEmpty()) { + return; // should not be here + } + files.sort((a, b) -> b.getLastOpDate().compareTo(a.getLastOpDate())); + + // Re-check VM status each iteration as per user's requirement + VmInstanceEO eo = Q.New(VmInstanceEO.class).eq(VmInstanceVO_.uuid, vmUuid).find(); + boolean vmActive = eo != null && eo.getDeleted() == null; + + if (vmActive) { + cleanupActiveVmHostFiles(files, now); + } else if (eo != null && eo.getDeleted() != null) { + cleanupDeletedVmHostFiles(files, now); + } else { + for (VmHostFileVO file : files) { + logger.info(String.format("deleting VmHostFileVO[uuid=%s] for expunged VM[uuid=%s]", + file.getUuid(), vmUuid)); + deleteVmHostFileFromDb(file.getUuid()); + } + } + }); + } + + private void cleanupActiveVmHostFiles(List files, long now) { + if (files.size() <= 1) { + return; + } + + VmHostFileVO latestWithContent = null; + for (VmHostFileVO file : files) { + if (hasContent(file.getUuid())) { + latestWithContent = file; + break; + } + } + + boolean allOlderThanOneDay = files.stream() + .allMatch(f -> now - f.getLastOpDate().getTime() > ONE_DAY_MS); + + for (int i = 1; i < files.size(); i++) { + VmHostFileVO file = files.get(i); + long age = now - file.getLastOpDate().getTime(); + + if (age <= ONE_DAY_MS) { + continue; + } + + if (allOlderThanOneDay && latestWithContent != null + && latestWithContent.getUuid().equals(file.getUuid()) + && !latestWithContent.getUuid().equals(files.get(0).getUuid())) { + continue; + } + + if (hasContent(file.getUuid())) { + boolean latestHasContent = hasContent(files.get(0).getUuid()); + if (!latestHasContent && latestWithContent != null + && latestWithContent.getUuid().equals(file.getUuid())) { + continue; + } + } + + boolean cmdSuccess = sendDeleteCommandToHost(file); + if (cmdSuccess) { + logger.info(String.format("deleting expired VmHostFileVO[uuid=%s, vm=%s, type=%s, age=%d hours]", + file.getUuid(), file.getVmInstanceUuid(), file.getType(), TimeUnit.MILLISECONDS.toHours(age))); + deleteVmHostFileFromDb(file.getUuid()); + } else { + logger.warn(String.format("failed to delete host file for VmHostFileVO[uuid=%s], skipping DB deletion", + file.getUuid())); + } + } + } + + private void cleanupDeletedVmHostFiles(List files, long now) { + for (int i = 0; i < files.size(); i++) { + VmHostFileVO file = files.get(i); + long age = now - file.getLastOpDate().getTime(); + + if (i == 0 && age <= NINETY_DAYS_MS) { + continue; + } + + if (age <= SEVEN_DAYS_MS) { + boolean cmdSuccess = sendDeleteCommandToHost(file); + if (!cmdSuccess) { + logger.warn(String.format( + "KVM delete failed for VmHostFileVO[uuid=%s] of deleted VM, lastOpDate within 7 days, skipping", + file.getUuid())); + continue; + } + } else { + sendDeleteCommandToHost(file); + } + + logger.info(String.format("deleting VmHostFileVO[uuid=%s] for deleted VM[uuid=%s, type=%s, age=%d days]", + file.getUuid(), file.getVmInstanceUuid(), file.getType(), TimeUnit.MILLISECONDS.toDays(age))); + deleteVmHostFileFromDb(file.getUuid()); + } + } + + /** + * resourceVO no longer exists: delete + * Other: do not delete + */ + private void cleanupExpiredVmHostBackupFiles() { + List allBackupFiles = Q.New(VmHostBackupFileVO.class).list(); + if (allBackupFiles.isEmpty()) { + return; + } + + Set existingResourceUuids = new HashSet<>(Q.New(ResourceVO.class) + .in(ResourceVO_.uuid, transformToSet(allBackupFiles, VmHostBackupFileVO::getResourceUuid)) + .select(ResourceVO_.uuid) + .listValues()); + + CollectionUtils.safeForEach(allBackupFiles, backupFile -> { + String resourceUuid = backupFile.getResourceUuid(); + if (!destinationMaker.isManagedByUs(resourceUuid)) { + return; + } + + if (existingResourceUuids.contains(resourceUuid)) { + return; + } + + logger.info(String.format( + "deleting VmHostBackupFileVO[uuid=%s] -> referenced resource[uuid=%s] no longer exists", + backupFile.getUuid(), resourceUuid)); + deleteVmHostBackupFileFromDb(backupFile.getUuid()); + }); + } + + /** + * Send a KVM delete command to the host for the given file. + * Returns true if the command succeeded or the host doesn't exist (allowing DB cleanup). + */ + private boolean sendDeleteCommandToHost(VmHostFileVO file) { + String hostUuid = file.getHostUuid(); + + // If host is gone (soft-deleted or expunged), allow direct DB cleanup + if (!Q.New(HostVO.class).eq(HostVO_.uuid, hostUuid).isExists()) { + return true; + } + + KVMAgentCommands.WriteVmHostFileContentCmd cmd = new KVMAgentCommands.WriteVmHostFileContentCmd(); + KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); + to.setPath(file.getPath()); + to.setType(file.getType().toString()); + to.setOperation(VmHostFileOperation.Delete.toString()); + cmd.setHostFiles(Collections.singletonList(to)); + + FutureReturnValueCompletion future = new FutureReturnValueCompletion(null); + new KvmCommandSender(hostUuid).send(cmd, KVMConstant.WRITE_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.WriteVmHostFileContentResponse rsp = + wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class); + return rsp.isSuccess() ? null : operr(ORG_ZSTACK_KVM_10163, "failed to delete host file[path=%s] on host[uuid=%s]", + file.getPath(), hostUuid); + }, future); + + future.await(KVM_CMD_TIMEOUT_MS); + return future.isSuccess(); + } + + private boolean hasContent(String fileUuid) { + return Q.New(VmHostFileContentVO.class).eq(VmHostFileContentVO_.uuid, fileUuid).isExists(); + } + + private void deleteVmHostFileFromDb(String fileUuid) { + VmHostFileDeletionMsg msg = new VmHostFileDeletionMsg(); + msg.setUuid(fileUuid); + msg.setForceDelete(false); + bus.makeLocalServiceId(msg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + MessageReply reply = bus.call(msg); + if (!reply.isSuccess()) { + logger.warn(String.format("failed to delete VmHostFileVO[uuid=%s] via VmHostFileDeletionMsg: %s", + fileUuid, reply.getError().getReadableDetails())); + } + } + + /** + * Deletes a {@link VmHostBackupFileVO} only through {@link VmHostBackupFileDeletionMsg} so related rows + * are cleaned in the handler/factory order. If the message fails, do not issue a second direct SQL delete: + * removing {@code VmHostBackupFileVO} alone could orphan ref tables that would then never be cleaned up. + */ + private void deleteVmHostBackupFileFromDb(String fileUuid) { + VmHostBackupFileDeletionMsg msg = new VmHostBackupFileDeletionMsg(); + msg.setUuid(fileUuid); + msg.setForceDelete(true); + bus.makeLocalServiceId(msg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + MessageReply reply = bus.call(msg); + if (!reply.isSuccess()) { + // No fallback DB delete: keep the row until a successful deletion message runs the full cleanup path. + logger.warn(String.format("failed to delete VmHostBackupFileVO[uuid=%s] via VmHostBackupFileDeletionMsg: %s", + fileUuid, reply.getError().getReadableDetails())); + } + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileMsg.java new file mode 100644 index 00000000000..18a0930d7fc --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileMsg.java @@ -0,0 +1,35 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.NeedReplyMessage; + +import java.util.List; + +public class BackupVmHostFileMsg extends NeedReplyMessage { + private String vmUuid; + private String hostUuid; + private List toResourceUuidList; + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public List getToResourceUuidList() { + return toResourceUuidList; + } + + public void setToResourceUuidList(List toResourceUuidList) { + this.toResourceUuidList = toResourceUuidList; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileOnHypervisorMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileOnHypervisorMsg.java new file mode 100644 index 00000000000..1ac815f8ff9 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileOnHypervisorMsg.java @@ -0,0 +1,29 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.vm.additions.VmHostFileBackupJob; + +import java.util.List; + +public class BackupVmHostFileOnHypervisorMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + private List vmHostFileBackupJobs; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public List getVmHostFileBackupJobs() { + return vmHostFileBackupJobs; + } + + public void setVmHostFileBackupJobs(List vmHostFileBackupJobs) { + this.vmHostFileBackupJobs = vmHostFileBackupJobs; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileOnHypervisorReply.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileOnHypervisorReply.java new file mode 100644 index 00000000000..f3268a43f83 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileOnHypervisorReply.java @@ -0,0 +1,6 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.MessageReply; + +public class BackupVmHostFileOnHypervisorReply extends MessageReply { +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileReply.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileReply.java new file mode 100644 index 00000000000..f2a22b3d920 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/BackupVmHostFileReply.java @@ -0,0 +1,17 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.MessageReply; + +import java.util.List; + +public class BackupVmHostFileReply extends MessageReply { + private List backupFileUuidList; + + public List getBackupFileUuidList() { + return backupFileUuidList; + } + + public void setBackupFileUuidList(List backupFileUuidList) { + this.backupFileUuidList = backupFileUuidList; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/CloneVmHostFileMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/CloneVmHostFileMsg.java new file mode 100644 index 00000000000..3e7817edbd9 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/CloneVmHostFileMsg.java @@ -0,0 +1,44 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.NeedReplyMessage; + +import java.util.List; + +public class CloneVmHostFileMsg extends NeedReplyMessage { + private String srcVmUuid; + private List dstVmUuidList; + private Boolean resetTpm; + private boolean ignoreSyncError = false; + + public String getSrcVmUuid() { + return srcVmUuid; + } + + public void setSrcVmUuid(String srcVmUuid) { + this.srcVmUuid = srcVmUuid; + } + + public List getDstVmUuidList() { + return dstVmUuidList; + } + + public void setDstVmUuidList(List dstVmUuidList) { + this.dstVmUuidList = dstVmUuidList; + } + + public Boolean getResetTpm() { + return resetTpm; + } + + public void setResetTpm(Boolean resetTpm) { + this.resetTpm = resetTpm; + } + + public boolean isIgnoreSyncError() { + return ignoreSyncError; + } + + public void setIgnoreSyncError(boolean ignoreSyncError) { + this.ignoreSyncError = ignoreSyncError; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/CloneVmHostFileReply.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/CloneVmHostFileReply.java new file mode 100644 index 00000000000..b29e604ed39 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/CloneVmHostFileReply.java @@ -0,0 +1,6 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.MessageReply; + +public class CloneVmHostFileReply extends MessageReply { +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java new file mode 100644 index 00000000000..93b21797f14 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java @@ -0,0 +1,69 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.NeedReplyMessage; + +public class SyncVmHostFilesFromHostMsg extends NeedReplyMessage { + private String hostUuid; + private String vmUuid; + private String nvRamPath; + private String tpmStateFolder; + private String syncReason; + private boolean syncToBackup; + private String backupResourceUuid; + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getNvRamPath() { + return nvRamPath; + } + + public void setNvRamPath(String nvRamPath) { + this.nvRamPath = nvRamPath; + } + + public String getTpmStateFolder() { + return tpmStateFolder; + } + + public void setTpmStateFolder(String tpmStateFolder) { + this.tpmStateFolder = tpmStateFolder; + } + + public String getSyncReason() { + return syncReason; + } + + public void setSyncReason(String syncReason) { + this.syncReason = syncReason; + } + + public boolean isSyncToBackup() { + return syncToBackup; + } + + public void setSyncToBackup(boolean syncToBackup) { + this.syncToBackup = syncToBackup; + } + + public String getBackupResourceUuid() { + return backupResourceUuid; + } + + public void setBackupResourceUuid(String backupResourceUuid) { + this.backupResourceUuid = backupResourceUuid; + } +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostReply.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostReply.java new file mode 100644 index 00000000000..e920c800985 --- /dev/null +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostReply.java @@ -0,0 +1,6 @@ +package org.zstack.kvm.vmfiles.message; + +import org.zstack.header.message.MessageReply; + +public class SyncVmHostFilesFromHostReply extends MessageReply { +} diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java index ec0b51711ba..7617e79f7d4 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java @@ -35,6 +35,7 @@ import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.host.*; import org.zstack.header.image.ImageBackupStorageRefInventory; +import org.zstack.header.image.ImageConstant; import org.zstack.header.image.ImageConstant.ImageMediaType; import org.zstack.header.image.ImageInventory; import org.zstack.header.image.ImageStatus; @@ -216,6 +217,7 @@ public static class CreateEmptyVolumeCmd extends AgentCommand { private String volumeUuid; @GrayVersion(value = "5.0.0") private String backingFile; + private String volumeFormat; public String getBackingFile() { return backingFile; @@ -225,6 +227,14 @@ public void setBackingFile(String backingFile) { this.backingFile = backingFile; } + public String getVolumeFormat() { + return volumeFormat; + } + + public void setVolumeFormat(String volumeFormat) { + this.volumeFormat = volumeFormat; + } + public String getInstallUrl() { return installUrl; } @@ -1052,6 +1062,10 @@ public String makeDataVolumeInstallUrl(String volUuid) { return PathUtil.join(self.getUrl(), PrimaryStoragePathMaker.makeDataVolumeInstallPath(volUuid)); } + public String makeNvRamVolumeInstallUrl(String volUuid) { + return PathUtil.join(self.getUrl(), PrimaryStoragePathMaker.makeNvRamVolumeInstallPath(volUuid)); + } + public boolean isCachedImageUrl(String path){ return path.startsWith(PathUtil.join(self.getUrl(), PrimaryStoragePathMaker.getCachedImageInstallDir())); } @@ -1346,7 +1360,7 @@ public void success(VolumeStats returnValue) { VolumeInventory vol = msg.getVolume(); vol.setInstallPath(returnValue.getInstallPath()); vol.setActualSize(returnValue.getActualSize()); - vol.setFormat(VolumeConstant.VOLUME_FORMAT_QCOW2); + vol.setFormat(returnValue.getFormat()); if (returnValue.getSize() != null) { vol.setSize(returnValue.getSize()); } @@ -1371,7 +1385,9 @@ public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final cmd.setAccountUuid(acntMgr.getOwnerAccountUuidOfResource(volume.getUuid())); if (volume.getInstallPath() != null && !volume.getInstallPath().equals("")) { cmd.setInstallUrl(volume.getInstallPath()); + cmd.setVolumeFormat(ImageConstant.QCOW2_FORMAT_STRING); } else { + cmd.setVolumeFormat(ImageConstant.QCOW2_FORMAT_STRING); if (VolumeType.Root.toString().equals(volume.getType())) { cmd.setInstallUrl(makeRootVolumeInstallUrl(volume)); } else if (VolumeType.Data.toString().equals(volume.getType())) { @@ -1390,7 +1406,9 @@ public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final httpCall(CREATE_EMPTY_VOLUME_PATH, hostUuid, cmd, CreateEmptyVolumeRsp.class, new ReturnValueCompletion(completion) { @Override public void success(CreateEmptyVolumeRsp returnValue) { - completion.success(new VolumeStats(cmd.getInstallUrl(), returnValue.actualSize, returnValue.size)); + final VolumeStats stats = new VolumeStats(cmd.getInstallUrl(), returnValue.actualSize, returnValue.size); + stats.setFormat(cmd.getVolumeFormat()); + completion.success(stats); } @Override diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java index a1341ef4449..43cb1277f88 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java @@ -1150,7 +1150,7 @@ public void run(MessageReply reply) { } volume.setInstallPath(cmd.getInstallUrl()); - volume.setFormat(VolumeConstant.VOLUME_FORMAT_QCOW2); + volume.setFormat(cmd.getVolumeFormat()); volume.setActualSize(rsp.actualSize); if (rsp.size != null) { volume.setSize(rsp.size); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java index 1a31b5b6b7e..41e75e2b79a 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java @@ -3,6 +3,7 @@ import org.zstack.core.upgrade.GrayVersion; import org.zstack.header.HasThreadContext; import org.zstack.header.core.validation.Validation; +import org.zstack.header.volume.VolumeConstant; import org.zstack.kvm.KVMAgentCommands; import org.zstack.kvm.KVMAgentCommands.AgentCommand; import org.zstack.kvm.KVMAgentCommands.AgentResponse; @@ -373,6 +374,7 @@ public abstract static class CreateVolumeCmd extends NfsPrimaryStorageAgentComma private String name; @GrayVersion(value = "5.0.0") private String volumeUuid; + private String volumeFormat = VolumeConstant.VOLUME_FORMAT_QCOW2; public String getInstallUrl() { return installUrl; @@ -404,7 +406,12 @@ public String getVolumeUuid() { public void setVolumeUuid(String uuid) { this.volumeUuid = uuid; } - + public String getVolumeFormat() { + return volumeFormat; + } + public void setVolumeFormat(String volumeFormat) { + this.volumeFormat = volumeFormat; + } } public static class CreateRootVolumeFromTemplateCmd extends CreateVolumeCmd { diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKvmHelper.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKvmHelper.java index a58456a47db..5de2022ebc1 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKvmHelper.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKvmHelper.java @@ -44,6 +44,10 @@ public static String makeCachedImageInstallUrl(PrimaryStorageInventory pinv, Ima return ImageCacheUtil.getImageCachePath(iminv, it -> PathUtil.join(pinv.getMountPath(), PrimaryStoragePathMaker.makeCachedImageInstallPath(iminv))); } + public static String makeNvRamVolumeInstallUrl(PrimaryStorageInventory pinv, String volUuid) { + return PathUtil.join(pinv.getMountPath(), PrimaryStoragePathMaker.makeNvRamVolumeInstallPath(volUuid)); + } + public static String getCachedImageDir(PrimaryStorageInventory pinv){ return PathUtil.join(pinv.getMountPath(), PrimaryStoragePathMaker.getCachedImageInstallDir()); } diff --git a/plugin/sharedMountPointPrimaryStorage/src/main/java/org/zstack/storage/primary/smp/KvmBackend.java b/plugin/sharedMountPointPrimaryStorage/src/main/java/org/zstack/storage/primary/smp/KvmBackend.java index 97a16810f69..32717992ebd 100755 --- a/plugin/sharedMountPointPrimaryStorage/src/main/java/org/zstack/storage/primary/smp/KvmBackend.java +++ b/plugin/sharedMountPointPrimaryStorage/src/main/java/org/zstack/storage/primary/smp/KvmBackend.java @@ -381,6 +381,7 @@ public static class CreateEmptyVolumeCmd extends AgentCmd { @GrayVersion(value = "5.0.0") public String volumeUuid; public String backingFile; + public String volumeFormat = VolumeConstant.VOLUME_FORMAT_QCOW2; } public static class CreateEmptyVolumeRsp extends AgentRsp { @@ -692,6 +693,10 @@ public String makeCachedImageInstallUrl(ImageInventory iminv) { return ImageCacheUtil.getImageCachePath(iminv, it -> PathUtil.join(self.getMountPath(), PrimaryStoragePathMaker.makeCachedImageInstallPath(iminv))); } + public String makeNvRamVolumeInstallUrl(String volUuid) { + return PathUtil.join(self.getMountPath(), PrimaryStoragePathMaker.makeNvRamVolumeInstallPath(volUuid)); + } + public String makeCachedImageInstallUrlFromImageUuidForTemplate(String imageUuid) { return PathUtil.join(self.getMountPath(), PrimaryStoragePathMaker.makeCachedImageInstallPathFromImageUuidForTemplate(imageUuid)); } @@ -1064,7 +1069,7 @@ public void success(AgentRsp returnValue) { CreateEmptyVolumeRsp rsp = (CreateEmptyVolumeRsp) returnValue; InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); volume.setInstallPath(cmd.installPath); - volume.setFormat(VolumeConstant.VOLUME_FORMAT_QCOW2); + volume.setFormat(cmd.volumeFormat); volume.setActualSize(rsp.actualSize); if (rsp.size != null) { volume.setSize(rsp.size); diff --git a/sdk/src/main/java/SourceClassMap.java b/sdk/src/main/java/SourceClassMap.java index 29c9197c37d..fd2eb989826 100644 --- a/sdk/src/main/java/SourceClassMap.java +++ b/sdk/src/main/java/SourceClassMap.java @@ -154,6 +154,9 @@ public class SourceClassMap { put("org.zstack.crypto.ccs.CCSCertificateInventory", "org.zstack.sdk.CCSCertificateInventory"); put("org.zstack.crypto.ccs.CCSCertificateUserRefInventory", "org.zstack.sdk.CCSCertificateUserRefInventory"); put("org.zstack.crypto.ccs.CCSCertificateUserState", "org.zstack.sdk.CCSCertificateUserState"); + put("org.zstack.crypto.keyprovider.api.RekeyFailedResource", "org.zstack.sdk.keyprovider.api.RekeyFailedResource"); + put("org.zstack.crypto.keyprovider.api.RekeyProviderResult", "org.zstack.sdk.keyprovider.api.RekeyProviderResult"); + put("org.zstack.crypto.keyprovider.api.RekeySkippedResource", "org.zstack.sdk.keyprovider.api.RekeySkippedResource"); put("org.zstack.crypto.securitymachine.thirdparty.aisino.AiSiNoSecretResourcePoolInventory", "org.zstack.sdk.AiSiNoSecretResourcePoolInventory"); put("org.zstack.crypto.securitymachine.thirdparty.csp.CSPSecretResourcePoolInventory", "org.zstack.sdk.CSPSecretResourcePoolInventory"); put("org.zstack.crypto.securitymachine.thirdparty.fiSec.FiSecSecretResourcePoolInventory", "org.zstack.sdk.FiSecSecretResourcePoolInventory"); @@ -328,6 +331,12 @@ public class SourceClassMap { put("org.zstack.header.image.ImageGroupInventory", "org.zstack.sdk.ImageGroupInventory"); put("org.zstack.header.image.ImageGroupRefInventory", "org.zstack.sdk.ImageGroupRefInventory"); put("org.zstack.header.image.ImageInventory", "org.zstack.sdk.ImageInventory"); + put("org.zstack.header.keyprovider.CertificateInfo", "org.zstack.sdk.CertificateInfo"); + put("org.zstack.header.keyprovider.KeyProviderInventory", "org.zstack.sdk.KeyProviderInventory"); + put("org.zstack.header.keyprovider.KmsIdentityInventory", "org.zstack.sdk.KmsIdentityInventory"); + put("org.zstack.header.keyprovider.KmsInventory", "org.zstack.sdk.KmsInventory"); + put("org.zstack.header.keyprovider.NkpInventory", "org.zstack.sdk.NkpInventory"); + put("org.zstack.header.keyprovider.NkpRestoreInfo", "org.zstack.sdk.NkpRestoreInfo"); put("org.zstack.header.longjob.LongJobInventory", "org.zstack.sdk.LongJobInventory"); put("org.zstack.header.longjob.LongJobState", "org.zstack.sdk.LongJobState"); put("org.zstack.header.managementnode.ManagementNodeInventory", "org.zstack.sdk.ManagementNodeInventory"); @@ -419,6 +428,8 @@ public class SourceClassMap { put("org.zstack.header.tag.TagPatternInventory", "org.zstack.sdk.TagPatternInventory"); put("org.zstack.header.tag.TagPatternType", "org.zstack.sdk.TagPatternType"); put("org.zstack.header.tag.UserTagInventory", "org.zstack.sdk.UserTagInventory"); + put("org.zstack.header.tpm.entity.TpmCapabilityView", "org.zstack.sdk.tpm.entity.TpmCapabilityView"); + put("org.zstack.header.tpm.entity.TpmInventory", "org.zstack.sdk.tpm.entity.TpmInventory"); put("org.zstack.header.vdpa.VmVdpaNicInventory", "org.zstack.sdk.VmVdpaNicInventory"); put("org.zstack.header.vipQos.VipQosInventory", "org.zstack.sdk.VipQosInventory"); put("org.zstack.header.vipQos.VpcSharedQosInventory", "org.zstack.sdk.VpcSharedQosInventory"); @@ -434,6 +445,7 @@ public class SourceClassMap { put("org.zstack.header.vm.VmPriorityConfigInventory", "org.zstack.sdk.VmPriorityConfigInventory"); put("org.zstack.header.vm.VmPriorityLevel", "org.zstack.sdk.VmPriorityLevel"); put("org.zstack.header.vm.VmSchedHistoryInventory", "org.zstack.sdk.VmSchedHistoryInventory"); + put("org.zstack.header.vm.additions.VmHostFileInventory", "org.zstack.sdk.vm.entity.VmHostFileInventory"); put("org.zstack.header.vm.cdrom.VmCdRomInventory", "org.zstack.sdk.VmCdRomInventory"); put("org.zstack.header.vm.devices.DeviceAddress", "org.zstack.sdk.DeviceAddress"); put("org.zstack.header.vm.devices.VmInstanceDeviceAddressArchiveInventory", "org.zstack.sdk.VmInstanceDeviceAddressArchiveInventory"); @@ -1051,6 +1063,7 @@ public class SourceClassMap { put("org.zstack.sdk.CephPrimaryStorageInventory", "org.zstack.storage.ceph.primary.CephPrimaryStorageInventory"); put("org.zstack.sdk.CephPrimaryStorageMonInventory", "org.zstack.storage.ceph.primary.CephPrimaryStorageMonInventory"); put("org.zstack.sdk.CephPrimaryStoragePoolInventory", "org.zstack.storage.ceph.primary.CephPrimaryStoragePoolInventory"); + put("org.zstack.sdk.CertificateInfo", "org.zstack.header.keyprovider.CertificateInfo"); put("org.zstack.sdk.CertificateInventory", "org.zstack.network.service.lb.CertificateInventory"); put("org.zstack.sdk.ChainInfo", "org.zstack.header.core.progress.ChainInfo"); put("org.zstack.sdk.ChronyServerInfo", "org.zstack.zops.ChronyServerInfo"); @@ -1237,6 +1250,9 @@ public class SourceClassMap { put("org.zstack.sdk.KVMCephVolumeTO", "org.zstack.storage.ceph.primary.KVMCephVolumeTO"); put("org.zstack.sdk.KVMHostInventory", "org.zstack.kvm.KVMHostInventory"); put("org.zstack.sdk.KVMIsoTO", "org.zstack.kvm.KVMIsoTO"); + put("org.zstack.sdk.KeyProviderInventory", "org.zstack.header.keyprovider.KeyProviderInventory"); + put("org.zstack.sdk.KmsIdentityInventory", "org.zstack.header.keyprovider.KmsIdentityInventory"); + put("org.zstack.sdk.KmsInventory", "org.zstack.header.keyprovider.KmsInventory"); put("org.zstack.sdk.KoAlSecretResourcePoolInventory", "org.zstack.crypto.securitymachine.thirdparty.koal.KoAlSecretResourcePoolInventory"); put("org.zstack.sdk.KvmCephCdRomTO", "org.zstack.storage.ceph.primary.KvmCephCdRomTO"); put("org.zstack.sdk.KvmCephIsoTO", "org.zstack.storage.ceph.primary.KvmCephIsoTO"); @@ -1357,6 +1373,8 @@ public class SourceClassMap { put("org.zstack.sdk.NfvInstOfferingInventory", "org.zstack.network.service.header.nfvinstgroup.NfvInstOfferingInventory"); put("org.zstack.sdk.NginxRedirectRule", "org.zstack.ai.NginxRedirectRule"); put("org.zstack.sdk.NicTO", "org.zstack.kvm.KVMAgentCommands$NicTO"); + put("org.zstack.sdk.NkpInventory", "org.zstack.header.keyprovider.NkpInventory"); + put("org.zstack.sdk.NkpRestoreInfo", "org.zstack.header.keyprovider.NkpRestoreInfo"); put("org.zstack.sdk.NormalIpRangeInventory", "org.zstack.header.network.l3.NormalIpRangeInventory"); put("org.zstack.sdk.NvmeLunHostRefInventory", "org.zstack.storage.device.nvme.NvmeLunHostRefInventory"); put("org.zstack.sdk.NvmeLunInventory", "org.zstack.storage.device.nvme.NvmeLunInventory"); @@ -1735,6 +1753,9 @@ public class SourceClassMap { put("org.zstack.sdk.identity.role.RoleState", "org.zstack.header.identity.role.RoleState"); put("org.zstack.sdk.identity.role.RoleStateEvent", "org.zstack.header.identity.role.RoleStateEvent"); put("org.zstack.sdk.identity.role.RoleType", "org.zstack.header.identity.role.RoleType"); + put("org.zstack.sdk.keyprovider.api.RekeyFailedResource", "org.zstack.crypto.keyprovider.api.RekeyFailedResource"); + put("org.zstack.sdk.keyprovider.api.RekeyProviderResult", "org.zstack.crypto.keyprovider.api.RekeyProviderResult"); + put("org.zstack.sdk.keyprovider.api.RekeySkippedResource", "org.zstack.crypto.keyprovider.api.RekeySkippedResource"); put("org.zstack.sdk.license.header.server.LicenseAuthorizedCapacityClientAddOnUsageView", "org.zstack.license.header.server.LicenseAuthorizedCapacityClientAddOnUsageView"); put("org.zstack.sdk.license.header.server.LicenseAuthorizedCapacityClientUsageView", "org.zstack.license.header.server.LicenseAuthorizedCapacityClientUsageView"); put("org.zstack.sdk.license.header.server.LicenseAuthorizedCapacityInventory", "org.zstack.license.header.server.LicenseAuthorizedCapacityInventory"); @@ -1783,6 +1804,9 @@ public class SourceClassMap { put("org.zstack.sdk.ticket.entity.TicketTypeInventory", "org.zstack.ticket.entity.TicketTypeInventory"); put("org.zstack.sdk.ticket.iam2.entity.IAM2TicketFlowCollectionInventory", "org.zstack.ticket.iam2.entity.IAM2TicketFlowCollectionInventory"); put("org.zstack.sdk.ticket.iam2.entity.IAM2TicketFlowInventory", "org.zstack.ticket.iam2.entity.IAM2TicketFlowInventory"); + put("org.zstack.sdk.tpm.entity.TpmCapabilityView", "org.zstack.header.tpm.entity.TpmCapabilityView"); + put("org.zstack.sdk.tpm.entity.TpmInventory", "org.zstack.header.tpm.entity.TpmInventory"); + put("org.zstack.sdk.vm.entity.VmHostFileInventory", "org.zstack.header.vm.additions.VmHostFileInventory"); put("org.zstack.sdk.zwatch.alarm.ActionParam", "org.zstack.zwatch.alarm.APICreateAlarmMsg$ActionParam"); put("org.zstack.sdk.zwatch.alarm.AlarmActionInventory", "org.zstack.zwatch.alarm.AlarmActionInventory"); put("org.zstack.sdk.zwatch.alarm.AlarmDataAckInventory", "org.zstack.zwatch.alarm.AlarmDataAckInventory"); diff --git a/sdk/src/main/java/org/zstack/heder/storage/volume/backup/CreateVmFromVolumeBackupAction.java b/sdk/src/main/java/org/zstack/heder/storage/volume/backup/CreateVmFromVolumeBackupAction.java index ebc015f3576..a9f67acf610 100644 --- a/sdk/src/main/java/org/zstack/heder/storage/volume/backup/CreateVmFromVolumeBackupAction.java +++ b/sdk/src/main/java/org/zstack/heder/storage/volume/backup/CreateVmFromVolumeBackupAction.java @@ -67,6 +67,9 @@ public Result throwExceptionIfError() { @Param(required = false, validValues = {"InstantStart","JustCreate","CreateStopped"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String strategy = "InstantStart"; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean resetTpm; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/CertificateInfo.java b/sdk/src/main/java/org/zstack/sdk/CertificateInfo.java new file mode 100644 index 00000000000..5dc36854ed1 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/CertificateInfo.java @@ -0,0 +1,55 @@ +package org.zstack.sdk; + + + +public class CertificateInfo { + + public java.lang.String subject; + public void setSubject(java.lang.String subject) { + this.subject = subject; + } + public java.lang.String getSubject() { + return this.subject; + } + + public java.lang.String issuer; + public void setIssuer(java.lang.String issuer) { + this.issuer = issuer; + } + public java.lang.String getIssuer() { + return this.issuer; + } + + public java.lang.String commonName; + public void setCommonName(java.lang.String commonName) { + this.commonName = commonName; + } + public java.lang.String getCommonName() { + return this.commonName; + } + + public java.util.List subjectAltNamesDns; + public void setSubjectAltNamesDns(java.util.List subjectAltNamesDns) { + this.subjectAltNamesDns = subjectAltNamesDns; + } + public java.util.List getSubjectAltNamesDns() { + return this.subjectAltNamesDns; + } + + public java.util.List subjectAltNamesIp; + public void setSubjectAltNamesIp(java.util.List subjectAltNamesIp) { + this.subjectAltNamesIp = subjectAltNamesIp; + } + public java.util.List getSubjectAltNamesIp() { + return this.subjectAltNamesIp; + } + + public java.sql.Timestamp expiredDate; + public void setExpiredDate(java.sql.Timestamp expiredDate) { + this.expiredDate = expiredDate; + } + public java.sql.Timestamp getExpiredDate() { + return this.expiredDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/CloneVmInstanceAction.java b/sdk/src/main/java/org/zstack/sdk/CloneVmInstanceAction.java index 68bb4ca0f6a..ed9e43c201b 100644 --- a/sdk/src/main/java/org/zstack/sdk/CloneVmInstanceAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CloneVmInstanceAction.java @@ -58,6 +58,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String hostUuid; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean resetTpm; + @Param(required = false) public java.util.List systemTags; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateOvnControllerVmAction.java b/sdk/src/main/java/org/zstack/sdk/CreateOvnControllerVmAction.java index c7ccc7e2c19..019d9d6bed5 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateOvnControllerVmAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateOvnControllerVmAction.java @@ -112,6 +112,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List diskAOs; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.Map devices; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java index a6c9c4cd2c4..9b4a1fed544 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java @@ -112,6 +112,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List diskAOs; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.Map devices; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java index 2ef24418b33..e1aab46b788 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java @@ -76,6 +76,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.Map dataVolumeSystemTags; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean resetTpm; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/KeyProviderInventory.java b/sdk/src/main/java/org/zstack/sdk/KeyProviderInventory.java new file mode 100644 index 00000000000..a2eef45777e --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/KeyProviderInventory.java @@ -0,0 +1,63 @@ +package org.zstack.sdk; + + + +public class KeyProviderInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String name; + public void setName(java.lang.String name) { + this.name = name; + } + public java.lang.String getName() { + return this.name; + } + + public java.lang.String description; + public void setDescription(java.lang.String description) { + this.description = description; + } + public java.lang.String getDescription() { + return this.description; + } + + public java.lang.String type; + public void setType(java.lang.String type) { + this.type = type; + } + public java.lang.String getType() { + return this.type; + } + + public boolean connected; + public void setConnected(boolean connected) { + this.connected = connected; + } + public boolean getConnected() { + return this.connected; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/KmsIdentityInventory.java b/sdk/src/main/java/org/zstack/sdk/KmsIdentityInventory.java new file mode 100644 index 00000000000..d621aa1d749 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/KmsIdentityInventory.java @@ -0,0 +1,71 @@ +package org.zstack.sdk; + + + +public class KmsIdentityInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String kmsUuid; + public void setKmsUuid(java.lang.String kmsUuid) { + this.kmsUuid = kmsUuid; + } + public java.lang.String getKmsUuid() { + return this.kmsUuid; + } + + public java.lang.String identityType; + public void setIdentityType(java.lang.String identityType) { + this.identityType = identityType; + } + public java.lang.String getIdentityType() { + return this.identityType; + } + + public java.lang.String clientCertPem; + public void setClientCertPem(java.lang.String clientCertPem) { + this.clientCertPem = clientCertPem; + } + public java.lang.String getClientCertPem() { + return this.clientCertPem; + } + + public java.lang.String csrPem; + public void setCsrPem(java.lang.String csrPem) { + this.csrPem = csrPem; + } + public java.lang.String getCsrPem() { + return this.csrPem; + } + + public java.sql.Timestamp certExpiredDate; + public void setCertExpiredDate(java.sql.Timestamp certExpiredDate) { + this.certExpiredDate = certExpiredDate; + } + public java.sql.Timestamp getCertExpiredDate() { + return this.certExpiredDate; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/KmsInventory.java b/sdk/src/main/java/org/zstack/sdk/KmsInventory.java new file mode 100644 index 00000000000..8da5d74fa61 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/KmsInventory.java @@ -0,0 +1,80 @@ +package org.zstack.sdk; + +import org.zstack.sdk.CertificateInfo; +import org.zstack.sdk.KmsIdentityInventory; + +public class KmsInventory extends org.zstack.sdk.KeyProviderInventory { + + public java.lang.String endpoint; + public void setEndpoint(java.lang.String endpoint) { + this.endpoint = endpoint; + } + public java.lang.String getEndpoint() { + return this.endpoint; + } + + public java.lang.Integer port; + public void setPort(java.lang.Integer port) { + this.port = port; + } + public java.lang.Integer getPort() { + return this.port; + } + + public java.lang.String kmipVersion; + public void setKmipVersion(java.lang.String kmipVersion) { + this.kmipVersion = kmipVersion; + } + public java.lang.String getKmipVersion() { + return this.kmipVersion; + } + + public java.lang.String username; + public void setUsername(java.lang.String username) { + this.username = username; + } + public java.lang.String getUsername() { + return this.username; + } + + public java.lang.String trustState; + public void setTrustState(java.lang.String trustState) { + this.trustState = trustState; + } + public java.lang.String getTrustState() { + return this.trustState; + } + + public java.lang.String activeIdentityUuid; + public void setActiveIdentityUuid(java.lang.String activeIdentityUuid) { + this.activeIdentityUuid = activeIdentityUuid; + } + public java.lang.String getActiveIdentityUuid() { + return this.activeIdentityUuid; + } + + public java.lang.String serverCertPem; + public void setServerCertPem(java.lang.String serverCertPem) { + this.serverCertPem = serverCertPem; + } + public java.lang.String getServerCertPem() { + return this.serverCertPem; + } + + public CertificateInfo serverCertInfo; + public void setServerCertInfo(CertificateInfo serverCertInfo) { + this.serverCertInfo = serverCertInfo; + } + public CertificateInfo getServerCertInfo() { + return this.serverCertInfo; + } + + public KmsIdentityInventory activeIdentity; + public void setActiveIdentity(KmsIdentityInventory activeIdentity) { + this.activeIdentity = activeIdentity; + } + public KmsIdentityInventory getActiveIdentity() { + return this.activeIdentity; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/NkpInventory.java b/sdk/src/main/java/org/zstack/sdk/NkpInventory.java new file mode 100644 index 00000000000..b76e38f3313 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/NkpInventory.java @@ -0,0 +1,39 @@ +package org.zstack.sdk; + + + +public class NkpInventory extends org.zstack.sdk.KeyProviderInventory { + + public java.lang.String kdf; + public void setKdf(java.lang.String kdf) { + this.kdf = kdf; + } + public java.lang.String getKdf() { + return this.kdf; + } + + public java.lang.String saltPolicy; + public void setSaltPolicy(java.lang.String saltPolicy) { + this.saltPolicy = saltPolicy; + } + public java.lang.String getSaltPolicy() { + return this.saltPolicy; + } + + public boolean backedUp; + public void setBackedUp(boolean backedUp) { + this.backedUp = backedUp; + } + public boolean getBackedUp() { + return this.backedUp; + } + + public java.lang.Integer currentVersion; + public void setCurrentVersion(java.lang.Integer currentVersion) { + this.currentVersion = currentVersion; + } + public java.lang.Integer getCurrentVersion() { + return this.currentVersion; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/NkpRestoreInfo.java b/sdk/src/main/java/org/zstack/sdk/NkpRestoreInfo.java new file mode 100644 index 00000000000..a9f108dab40 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/NkpRestoreInfo.java @@ -0,0 +1,63 @@ +package org.zstack.sdk; + + + +public class NkpRestoreInfo { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String name; + public void setName(java.lang.String name) { + this.name = name; + } + public java.lang.String getName() { + return this.name; + } + + public java.lang.String description; + public void setDescription(java.lang.String description) { + this.description = description; + } + public java.lang.String getDescription() { + return this.description; + } + + public java.lang.String kdf; + public void setKdf(java.lang.String kdf) { + this.kdf = kdf; + } + public java.lang.String getKdf() { + return this.kdf; + } + + public java.lang.String saltPolicy; + public void setSaltPolicy(java.lang.String saltPolicy) { + this.saltPolicy = saltPolicy; + } + public java.lang.String getSaltPolicy() { + return this.saltPolicy; + } + + public java.lang.Integer currentVersion; + public void setCurrentVersion(java.lang.Integer currentVersion) { + this.currentVersion = currentVersion; + } + public java.lang.Integer getCurrentVersion() { + return this.currentVersion; + } + + public java.lang.Long backupTime; + public void setBackupTime(java.lang.Long backupTime) { + this.backupTime = backupTime; + } + public java.lang.Long getBackupTime() { + return this.backupTime; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/QueryKeyProviderAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/QueryKeyProviderAction.java new file mode 100644 index 00000000000..b47c9b694eb --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/QueryKeyProviderAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk.keyprovider.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryKeyProviderAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.api.QueryKeyProviderResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.api.QueryKeyProviderResult value = res.getResult(org.zstack.sdk.keyprovider.api.QueryKeyProviderResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.api.QueryKeyProviderResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/key-providers"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/QueryKeyProviderResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/QueryKeyProviderResult.java new file mode 100644 index 00000000000..b331fb5348a --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/QueryKeyProviderResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk.keyprovider.api; + + + +public class QueryKeyProviderResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyFailedResource.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyFailedResource.java new file mode 100644 index 00000000000..293d667dd6a --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyFailedResource.java @@ -0,0 +1,39 @@ +package org.zstack.sdk.keyprovider.api; + + + +public class RekeyFailedResource { + + public java.lang.Long keyRefId; + public void setKeyRefId(java.lang.Long keyRefId) { + this.keyRefId = keyRefId; + } + public java.lang.Long getKeyRefId() { + return this.keyRefId; + } + + public java.lang.String resourceType; + public void setResourceType(java.lang.String resourceType) { + this.resourceType = resourceType; + } + public java.lang.String getResourceType() { + return this.resourceType; + } + + public java.lang.String resourceUuid; + public void setResourceUuid(java.lang.String resourceUuid) { + this.resourceUuid = resourceUuid; + } + public java.lang.String getResourceUuid() { + return this.resourceUuid; + } + + public java.lang.String reason; + public void setReason(java.lang.String reason) { + this.reason = reason; + } + public java.lang.String getReason() { + return this.reason; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyKeyProviderRefsAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyKeyProviderRefsAction.java new file mode 100644 index 00000000000..0d476612170 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyKeyProviderRefsAction.java @@ -0,0 +1,113 @@ +package org.zstack.sdk.keyprovider.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class RekeyKeyProviderRefsAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.api.RekeyKeyProviderRefsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, nonempty = true, nullElements = false, emptyString = false, noTrim = false) + public java.util.List refIds; + + @Param(required = false, nonempty = true, nullElements = false, emptyString = false, noTrim = false) + public java.util.List resourceUuids; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String resourceType; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String providerUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public boolean rekeyAll = false; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.api.RekeyKeyProviderRefsResult value = res.getResult(org.zstack.sdk.keyprovider.api.RekeyKeyProviderRefsResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.api.RekeyKeyProviderRefsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/{providerUuid}/rekey"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "rekeyKeyProviderRefs"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyKeyProviderRefsResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyKeyProviderRefsResult.java new file mode 100644 index 00000000000..32a5c938b10 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyKeyProviderRefsResult.java @@ -0,0 +1,46 @@ +package org.zstack.sdk.keyprovider.api; + + + +public class RekeyKeyProviderRefsResult { + public int totalCount; + public void setTotalCount(int totalCount) { + this.totalCount = totalCount; + } + public int getTotalCount() { + return this.totalCount; + } + + public int successCount; + public void setSuccessCount(int successCount) { + this.successCount = successCount; + } + public int getSuccessCount() { + return this.successCount; + } + + public int skippedCount; + public void setSkippedCount(int skippedCount) { + this.skippedCount = skippedCount; + } + public int getSkippedCount() { + return this.skippedCount; + } + + public int failedCount; + public void setFailedCount(int failedCount) { + this.failedCount = failedCount; + } + public int getFailedCount() { + return this.failedCount; + } + + public java.util.List providerResults; + public void setProviderResults(java.util.List providerResults) { + this.providerResults = providerResults; + } + public java.util.List getProviderResults() { + return this.providerResults; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyProviderResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyProviderResult.java new file mode 100644 index 00000000000..bbebfaa0fe0 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeyProviderResult.java @@ -0,0 +1,71 @@ +package org.zstack.sdk.keyprovider.api; + + + +public class RekeyProviderResult { + + public java.lang.String providerUuid; + public void setProviderUuid(java.lang.String providerUuid) { + this.providerUuid = providerUuid; + } + public java.lang.String getProviderUuid() { + return this.providerUuid; + } + + public java.lang.String providerName; + public void setProviderName(java.lang.String providerName) { + this.providerName = providerName; + } + public java.lang.String getProviderName() { + return this.providerName; + } + + public int totalRefCount; + public void setTotalRefCount(int totalRefCount) { + this.totalRefCount = totalRefCount; + } + public int getTotalRefCount() { + return this.totalRefCount; + } + + public int successRefCount; + public void setSuccessRefCount(int successRefCount) { + this.successRefCount = successRefCount; + } + public int getSuccessRefCount() { + return this.successRefCount; + } + + public int skippedRefCount; + public void setSkippedRefCount(int skippedRefCount) { + this.skippedRefCount = skippedRefCount; + } + public int getSkippedRefCount() { + return this.skippedRefCount; + } + + public int failedRefCount; + public void setFailedRefCount(int failedRefCount) { + this.failedRefCount = failedRefCount; + } + public int getFailedRefCount() { + return this.failedRefCount; + } + + public java.util.List skippedResources; + public void setSkippedResources(java.util.List skippedResources) { + this.skippedResources = skippedResources; + } + public java.util.List getSkippedResources() { + return this.skippedResources; + } + + public java.util.List failedResources; + public void setFailedResources(java.util.List failedResources) { + this.failedResources = failedResources; + } + public java.util.List getFailedResources() { + return this.failedResources; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeySkippedResource.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeySkippedResource.java new file mode 100644 index 00000000000..70dce8675bf --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/api/RekeySkippedResource.java @@ -0,0 +1,39 @@ +package org.zstack.sdk.keyprovider.api; + + + +public class RekeySkippedResource { + + public java.lang.Long keyRefId; + public void setKeyRefId(java.lang.Long keyRefId) { + this.keyRefId = keyRefId; + } + public java.lang.Long getKeyRefId() { + return this.keyRefId; + } + + public java.lang.String resourceType; + public void setResourceType(java.lang.String resourceType) { + this.resourceType = resourceType; + } + public java.lang.String getResourceType() { + return this.resourceType; + } + + public java.lang.String resourceUuid; + public void setResourceUuid(java.lang.String resourceUuid) { + this.resourceUuid = resourceUuid; + } + public java.lang.String getResourceUuid() { + return this.resourceUuid; + } + + public java.lang.String reason; + public void setReason(java.lang.String reason) { + this.reason = reason; + } + public java.lang.String getReason() { + return this.reason; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/CreateKmsAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/CreateKmsAction.java new file mode 100644 index 00000000000..b70b28d4935 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/CreateKmsAction.java @@ -0,0 +1,125 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class CreateKmsAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.CreateKmsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String endpoint; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, numberRange = {1L,65535L}, noTrim = false) + public java.lang.Integer port; + + @Param(required = false, validValues = {"1.0","1.1","1.2","1.3","1.4","2.0","2.1"}, maxLength = 32, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String kmipVersion = "1.2"; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String username; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String password; + + @Param(required = true, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String name; + + @Param(required = false, maxLength = 2048, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String description; + + @Param(required = false) + public java.lang.String resourceUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.List tagUuids; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.CreateKmsResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.CreateKmsResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.CreateKmsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "POST"; + info.path = "/key-providers/kms"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "params"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/CreateKmsResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/CreateKmsResult.java new file mode 100644 index 00000000000..ae1e24307a1 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/CreateKmsResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.KeyProviderInventory; + +public class CreateKmsResult { + public KeyProviderInventory inventory; + public void setInventory(KeyProviderInventory inventory) { + this.inventory = inventory; + } + public KeyProviderInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/DeleteKmsAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/DeleteKmsAction.java new file mode 100644 index 00000000000..fd6bc7c013d --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/DeleteKmsAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class DeleteKmsAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.DeleteKmsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String uuid; + + @Param(required = false) + public java.lang.String deleteMode = "Permissive"; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.DeleteKmsResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.DeleteKmsResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.DeleteKmsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "DELETE"; + info.path = "/key-providers/kms/{uuid}"; + info.needSession = true; + info.needPoll = true; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/DeleteKmsResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/DeleteKmsResult.java new file mode 100644 index 00000000000..a88e9a3559e --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/DeleteKmsResult.java @@ -0,0 +1,7 @@ +package org.zstack.sdk.keyprovider.kms.api; + + + +public class DeleteKmsResult { + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/GetKmsServerCertFromKmsAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/GetKmsServerCertFromKmsAction.java new file mode 100644 index 00000000000..3cbac019b87 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/GetKmsServerCertFromKmsAction.java @@ -0,0 +1,101 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class GetKmsServerCertFromKmsAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.GetKmsServerCertFromKmsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.GetKmsServerCertFromKmsResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.GetKmsServerCertFromKmsResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.GetKmsServerCertFromKmsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/kms/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "getKmsServerCertFromKms"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/GetKmsServerCertFromKmsResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/GetKmsServerCertFromKmsResult.java new file mode 100644 index 00000000000..08a835fb0a5 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/GetKmsServerCertFromKmsResult.java @@ -0,0 +1,30 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.CertificateInfo; + +public class GetKmsServerCertFromKmsResult { + public java.lang.String serverCertPem; + public void setServerCertPem(java.lang.String serverCertPem) { + this.serverCertPem = serverCertPem; + } + public java.lang.String getServerCertPem() { + return this.serverCertPem; + } + + public boolean selfSigned; + public void setSelfSigned(boolean selfSigned) { + this.selfSigned = selfSigned; + } + public boolean getSelfSigned() { + return this.selfSigned; + } + + public CertificateInfo serverCertInfo; + public void setServerCertInfo(CertificateInfo serverCertInfo) { + this.serverCertInfo = serverCertInfo; + } + public CertificateInfo getServerCertInfo() { + return this.serverCertInfo; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/QueryKmsAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/QueryKmsAction.java new file mode 100644 index 00000000000..5b66b9f01cb --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/QueryKmsAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryKmsAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.QueryKmsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.QueryKmsResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.QueryKmsResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.QueryKmsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/key-providers/kms"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/QueryKmsResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/QueryKmsResult.java new file mode 100644 index 00000000000..d0d76d2f982 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/QueryKmsResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk.keyprovider.kms.api; + + + +public class QueryKmsResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UpdateKmsAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UpdateKmsAction.java new file mode 100644 index 00000000000..54423cdf317 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UpdateKmsAction.java @@ -0,0 +1,119 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UpdateKmsAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.UpdateKmsResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String endpoint; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = false, numberRange = {1L,65535L}, noTrim = false) + public java.lang.Integer port; + + @Param(required = false, validValues = {"1.0","1.1","1.2","1.3","1.4","2.0","2.1"}, maxLength = 32, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String kmipVersion; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String username; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String password; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = false, maxLength = 2048, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String description; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.UpdateKmsResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.UpdateKmsResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.UpdateKmsResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/kms/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "updateKms"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UpdateKmsResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UpdateKmsResult.java new file mode 100644 index 00000000000..8f48d409109 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UpdateKmsResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.KeyProviderInventory; + +public class UpdateKmsResult { + public KeyProviderInventory inventory; + public void setInventory(KeyProviderInventory inventory) { + this.inventory = inventory; + } + public KeyProviderInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientCsrAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientCsrAction.java new file mode 100644 index 00000000000..5633b9e053c --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientCsrAction.java @@ -0,0 +1,107 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UploadKmsClientCsrAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.UploadKmsClientCsrResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String csrPem; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String csrKeyPem; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.UploadKmsClientCsrResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.UploadKmsClientCsrResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.UploadKmsClientCsrResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/kms/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "uploadKmsClientCsr"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientCsrResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientCsrResult.java new file mode 100644 index 00000000000..562abefd6f9 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientCsrResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.KmsIdentityInventory; + +public class UploadKmsClientCsrResult { + public KmsIdentityInventory inventory; + public void setInventory(KmsIdentityInventory inventory) { + this.inventory = inventory; + } + public KmsIdentityInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientIdentityAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientIdentityAction.java new file mode 100644 index 00000000000..1e67dedad41 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientIdentityAction.java @@ -0,0 +1,110 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UploadKmsClientIdentityAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.UploadKmsClientIdentityResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = true, validValues = {"PLATFORM","UPLOADED","CSR"}, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String identityType; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String kmsClientCertPem; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String kmsClientKeyPem; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.UploadKmsClientIdentityResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.UploadKmsClientIdentityResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.UploadKmsClientIdentityResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/kms/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "uploadKmsClientIdentity"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientIdentityResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientIdentityResult.java new file mode 100644 index 00000000000..e71e7a51c1e --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientIdentityResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.KmsIdentityInventory; + +public class UploadKmsClientIdentityResult { + public KmsIdentityInventory inventory; + public void setInventory(KmsIdentityInventory inventory) { + this.inventory = inventory; + } + public KmsIdentityInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientSignedCertAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientSignedCertAction.java new file mode 100644 index 00000000000..9a8cb2f46df --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientSignedCertAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UploadKmsClientSignedCertAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.UploadKmsClientSignedCertResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String signedClientCertPem; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.UploadKmsClientSignedCertResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.UploadKmsClientSignedCertResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.UploadKmsClientSignedCertResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/kms/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "uploadKmsClientSignedCert"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientSignedCertResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientSignedCertResult.java new file mode 100644 index 00000000000..92ada658c38 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsClientSignedCertResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.KmsIdentityInventory; + +public class UploadKmsClientSignedCertResult { + public KmsIdentityInventory inventory; + public void setInventory(KmsIdentityInventory inventory) { + this.inventory = inventory; + } + public KmsIdentityInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsServerCertAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsServerCertAction.java new file mode 100644 index 00000000000..26303212c65 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsServerCertAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UploadKmsServerCertAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.kms.api.UploadKmsServerCertResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String serverCertPem; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.kms.api.UploadKmsServerCertResult value = res.getResult(org.zstack.sdk.keyprovider.kms.api.UploadKmsServerCertResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.kms.api.UploadKmsServerCertResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/kms/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "uploadKmsServerCert"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsServerCertResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsServerCertResult.java new file mode 100644 index 00000000000..7a0101c99c4 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/kms/api/UploadKmsServerCertResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.kms.api; + +import org.zstack.sdk.KmsInventory; + +public class UploadKmsServerCertResult { + public KmsInventory inventory; + public void setInventory(KmsInventory inventory) { + this.inventory = inventory; + } + public KmsInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/BackupNkpAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/BackupNkpAction.java new file mode 100644 index 00000000000..2141553cd8f --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/BackupNkpAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class BackupNkpAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.BackupNkpResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String password; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.BackupNkpResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.BackupNkpResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.BackupNkpResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/nkp/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "backupNkp"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/BackupNkpResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/BackupNkpResult.java new file mode 100644 index 00000000000..a8880061e50 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/BackupNkpResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.nkp.api; + + + +public class BackupNkpResult { + public java.lang.String content; + public void setContent(java.lang.String content) { + this.content = content; + } + public java.lang.String getContent() { + return this.content; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/CreateNkpAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/CreateNkpAction.java new file mode 100644 index 00000000000..36e7ef1906a --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/CreateNkpAction.java @@ -0,0 +1,116 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class CreateNkpAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.CreateNkpResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, validValues = {"HKDF-SHA256"}, maxLength = 64, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String kdf = "HKDF-SHA256"; + + @Param(required = false, validValues = {"providerName"}, maxLength = 64, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String saltPolicy = "providerName"; + + @Param(required = true, maxLength = 255, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String name; + + @Param(required = false, maxLength = 2048, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String description; + + @Param(required = false) + public java.lang.String resourceUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.List tagUuids; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.CreateNkpResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.CreateNkpResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.CreateNkpResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "POST"; + info.path = "/key-providers/nkp"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "params"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/CreateNkpResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/CreateNkpResult.java new file mode 100644 index 00000000000..e00c3112235 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/CreateNkpResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import org.zstack.sdk.KeyProviderInventory; + +public class CreateNkpResult { + public KeyProviderInventory inventory; + public void setInventory(KeyProviderInventory inventory) { + this.inventory = inventory; + } + public KeyProviderInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/DeleteNkpAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/DeleteNkpAction.java new file mode 100644 index 00000000000..643edf52008 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/DeleteNkpAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class DeleteNkpAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.DeleteNkpResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String uuid; + + @Param(required = false) + public java.lang.String deleteMode = "Permissive"; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.DeleteNkpResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.DeleteNkpResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.DeleteNkpResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "DELETE"; + info.path = "/key-providers/nkp/{uuid}"; + info.needSession = true; + info.needPoll = true; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/DeleteNkpResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/DeleteNkpResult.java new file mode 100644 index 00000000000..3dbd48e495b --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/DeleteNkpResult.java @@ -0,0 +1,7 @@ +package org.zstack.sdk.keyprovider.nkp.api; + + + +public class DeleteNkpResult { + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/ParseNkpRestoreAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/ParseNkpRestoreAction.java new file mode 100644 index 00000000000..9e8fc6ada51 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/ParseNkpRestoreAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class ParseNkpRestoreAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.ParseNkpRestoreResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String contentBase64; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String password; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.ParseNkpRestoreResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.ParseNkpRestoreResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.ParseNkpRestoreResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/nkp/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "parseNkpRestore"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/ParseNkpRestoreResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/ParseNkpRestoreResult.java new file mode 100644 index 00000000000..5b5ef838c90 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/ParseNkpRestoreResult.java @@ -0,0 +1,30 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import org.zstack.sdk.NkpRestoreInfo; + +public class ParseNkpRestoreResult { + public NkpRestoreInfo restoreInfo; + public void setRestoreInfo(NkpRestoreInfo restoreInfo) { + this.restoreInfo = restoreInfo; + } + public NkpRestoreInfo getRestoreInfo() { + return this.restoreInfo; + } + + public java.lang.String code; + public void setCode(java.lang.String code) { + this.code = code; + } + public java.lang.String getCode() { + return this.code; + } + + public java.lang.String reason; + public void setReason(java.lang.String reason) { + this.reason = reason; + } + public java.lang.String getReason() { + return this.reason; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/QueryNkpAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/QueryNkpAction.java new file mode 100644 index 00000000000..94e6cd6ff63 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/QueryNkpAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryNkpAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.QueryNkpResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.QueryNkpResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.QueryNkpResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.QueryNkpResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/key-providers/nkp"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/QueryNkpResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/QueryNkpResult.java new file mode 100644 index 00000000000..3a1b8f44abd --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/QueryNkpResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk.keyprovider.nkp.api; + + + +public class QueryNkpResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/RestoreNkpAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/RestoreNkpAction.java new file mode 100644 index 00000000000..5e92616c46f --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/RestoreNkpAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class RestoreNkpAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.RestoreNkpResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String contentBase64; + + @Param(required = false, maxLength = 255, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String password; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.RestoreNkpResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.RestoreNkpResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.RestoreNkpResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/nkp/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "restoreNkp"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/RestoreNkpResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/RestoreNkpResult.java new file mode 100644 index 00000000000..812f84e8fa9 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/RestoreNkpResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import org.zstack.sdk.NkpInventory; + +public class RestoreNkpResult { + public NkpInventory inventory; + public void setInventory(NkpInventory inventory) { + this.inventory = inventory; + } + public NkpInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/UpdateNkpAction.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/UpdateNkpAction.java new file mode 100644 index 00000000000..cf81539ecd6 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/UpdateNkpAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UpdateNkpAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.keyprovider.nkp.api.UpdateNkpResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = false, noTrim = false) + public java.lang.String uuid; + + @Param(required = false, maxLength = 2048, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String description; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.keyprovider.nkp.api.UpdateNkpResult value = res.getResult(org.zstack.sdk.keyprovider.nkp.api.UpdateNkpResult.class); + ret.value = value == null ? new org.zstack.sdk.keyprovider.nkp.api.UpdateNkpResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/key-providers/nkp/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "updateNkp"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/UpdateNkpResult.java b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/UpdateNkpResult.java new file mode 100644 index 00000000000..b7302e2c291 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/keyprovider/nkp/api/UpdateNkpResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.keyprovider.nkp.api; + +import org.zstack.sdk.KeyProviderInventory; + +public class UpdateNkpResult { + public KeyProviderInventory inventory; + public void setInventory(KeyProviderInventory inventory) { + this.inventory = inventory; + } + public KeyProviderInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/AddTpmAction.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/AddTpmAction.java new file mode 100644 index 00000000000..53f03a70614 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/AddTpmAction.java @@ -0,0 +1,110 @@ +package org.zstack.sdk.tpm.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class AddTpmAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.tpm.api.AddTpmResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, maxLength = 32, minLength = 32, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String keyProviderUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String vmInstanceUuid; + + @Param(required = false) + public java.lang.String resourceUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.List tagUuids; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.tpm.api.AddTpmResult value = res.getResult(org.zstack.sdk.tpm.api.AddTpmResult.class); + ret.value = value == null ? new org.zstack.sdk.tpm.api.AddTpmResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "POST"; + info.path = "/tpms"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "params"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/AddTpmResult.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/AddTpmResult.java new file mode 100644 index 00000000000..05effc20779 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/AddTpmResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.tpm.api; + +import org.zstack.sdk.tpm.entity.TpmInventory; + +public class AddTpmResult { + public TpmInventory inventory; + public void setInventory(TpmInventory inventory) { + this.inventory = inventory; + } + public TpmInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/GetTpmCapabilityAction.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/GetTpmCapabilityAction.java new file mode 100644 index 00000000000..f9e702fca70 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/GetTpmCapabilityAction.java @@ -0,0 +1,98 @@ +package org.zstack.sdk.tpm.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class GetTpmCapabilityAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.tpm.api.GetTpmCapabilityResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String tpmUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String vmInstanceUuid; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.tpm.api.GetTpmCapabilityResult value = res.getResult(org.zstack.sdk.tpm.api.GetTpmCapabilityResult.class); + ret.value = value == null ? new org.zstack.sdk.tpm.api.GetTpmCapabilityResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/tpms/capability"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/GetTpmCapabilityResult.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/GetTpmCapabilityResult.java new file mode 100644 index 00000000000..66d43f8a70d --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/GetTpmCapabilityResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.tpm.api; + +import org.zstack.sdk.tpm.entity.TpmCapabilityView; + +public class GetTpmCapabilityResult { + public TpmCapabilityView inventory; + public void setInventory(TpmCapabilityView inventory) { + this.inventory = inventory; + } + public TpmCapabilityView getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/QueryTpmAction.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/QueryTpmAction.java new file mode 100644 index 00000000000..5a5b0919a1d --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/QueryTpmAction.java @@ -0,0 +1,75 @@ +package org.zstack.sdk.tpm.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class QueryTpmAction extends QueryAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.tpm.api.QueryTpmResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.tpm.api.QueryTpmResult value = res.getResult(org.zstack.sdk.tpm.api.QueryTpmResult.class); + ret.value = value == null ? new org.zstack.sdk.tpm.api.QueryTpmResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "GET"; + info.path = "/tpms"; + info.needSession = true; + info.needPoll = false; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/QueryTpmResult.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/QueryTpmResult.java new file mode 100644 index 00000000000..1414409e00a --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/QueryTpmResult.java @@ -0,0 +1,22 @@ +package org.zstack.sdk.tpm.api; + + + +public class QueryTpmResult { + public java.util.List inventories; + public void setInventories(java.util.List inventories) { + this.inventories = inventories; + } + public java.util.List getInventories() { + return this.inventories; + } + + public java.lang.Long total; + public void setTotal(java.lang.Long total) { + this.total = total; + } + public java.lang.Long getTotal() { + return this.total; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/RemoveTpmAction.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/RemoveTpmAction.java new file mode 100644 index 00000000000..9f4414f04fa --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/RemoveTpmAction.java @@ -0,0 +1,107 @@ +package org.zstack.sdk.tpm.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class RemoveTpmAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.tpm.api.RemoveTpmResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String tpmUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String vmInstanceUuid; + + @Param(required = false) + public java.lang.String deleteMode = "Permissive"; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.tpm.api.RemoveTpmResult value = res.getResult(org.zstack.sdk.tpm.api.RemoveTpmResult.class); + ret.value = value == null ? new org.zstack.sdk.tpm.api.RemoveTpmResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "DELETE"; + info.path = "/tpms"; + info.needSession = true; + info.needPoll = true; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/RemoveTpmResult.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/RemoveTpmResult.java new file mode 100644 index 00000000000..93f74261a83 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/RemoveTpmResult.java @@ -0,0 +1,7 @@ +package org.zstack.sdk.tpm.api; + + + +public class RemoveTpmResult { + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/UpdateTpmAction.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/UpdateTpmAction.java new file mode 100644 index 00000000000..4f5478db13b --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/UpdateTpmAction.java @@ -0,0 +1,107 @@ +package org.zstack.sdk.tpm.api; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class UpdateTpmAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.tpm.api.UpdateTpmResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s, globalErrorCode: %s]", error.code, error.description, error.details, error.globalErrorCode) + ); + } + + return this; + } + } + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String vmInstanceUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String tpmUuid; + + @Param(required = false, maxLength = 32, minLength = 32, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String keyProviderUuid; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.tpm.api.UpdateTpmResult value = res.getResult(org.zstack.sdk.tpm.api.UpdateTpmResult.class); + ret.value = value == null ? new org.zstack.sdk.tpm.api.UpdateTpmResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/tpms"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "updateTpm"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/api/UpdateTpmResult.java b/sdk/src/main/java/org/zstack/sdk/tpm/api/UpdateTpmResult.java new file mode 100644 index 00000000000..1157fd732cb --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/api/UpdateTpmResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk.tpm.api; + +import org.zstack.sdk.tpm.entity.TpmInventory; + +public class UpdateTpmResult { + public TpmInventory inventory; + public void setInventory(TpmInventory inventory) { + this.inventory = inventory; + } + public TpmInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/entity/TpmCapabilityView.java b/sdk/src/main/java/org/zstack/sdk/tpm/entity/TpmCapabilityView.java new file mode 100644 index 00000000000..38fa336e3d5 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/entity/TpmCapabilityView.java @@ -0,0 +1,79 @@ +package org.zstack.sdk.tpm.entity; + + + +public class TpmCapabilityView { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String name; + public void setName(java.lang.String name) { + this.name = name; + } + public java.lang.String getName() { + return this.name; + } + + public java.lang.String vmInstanceUuid; + public void setVmInstanceUuid(java.lang.String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + public java.lang.String getVmInstanceUuid() { + return this.vmInstanceUuid; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + + public java.util.List fileRefs; + public void setFileRefs(java.util.List fileRefs) { + this.fileRefs = fileRefs; + } + public java.util.List getFileRefs() { + return this.fileRefs; + } + + public java.lang.String edkVersion; + public void setEdkVersion(java.lang.String edkVersion) { + this.edkVersion = edkVersion; + } + public java.lang.String getEdkVersion() { + return this.edkVersion; + } + + public java.lang.String swtpmVersion; + public void setSwtpmVersion(java.lang.String swtpmVersion) { + this.swtpmVersion = swtpmVersion; + } + public java.lang.String getSwtpmVersion() { + return this.swtpmVersion; + } + + public boolean resetTpmAfterVmCloneConfig; + public void setResetTpmAfterVmCloneConfig(boolean resetTpmAfterVmCloneConfig) { + this.resetTpmAfterVmCloneConfig = resetTpmAfterVmCloneConfig; + } + public boolean getResetTpmAfterVmCloneConfig() { + return this.resetTpmAfterVmCloneConfig; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/tpm/entity/TpmInventory.java b/sdk/src/main/java/org/zstack/sdk/tpm/entity/TpmInventory.java new file mode 100644 index 00000000000..538b12182ca --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/tpm/entity/TpmInventory.java @@ -0,0 +1,47 @@ +package org.zstack.sdk.tpm.entity; + + + +public class TpmInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String name; + public void setName(java.lang.String name) { + this.name = name; + } + public java.lang.String getName() { + return this.name; + } + + public java.lang.String vmInstanceUuid; + public void setVmInstanceUuid(java.lang.String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + public java.lang.String getVmInstanceUuid() { + return this.vmInstanceUuid; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/vm/entity/VmHostFileInventory.java b/sdk/src/main/java/org/zstack/sdk/vm/entity/VmHostFileInventory.java new file mode 100644 index 00000000000..555cf4b8d45 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/vm/entity/VmHostFileInventory.java @@ -0,0 +1,87 @@ +package org.zstack.sdk.vm.entity; + + + +public class VmHostFileInventory { + + public java.lang.String uuid; + public void setUuid(java.lang.String uuid) { + this.uuid = uuid; + } + public java.lang.String getUuid() { + return this.uuid; + } + + public java.lang.String vmInstanceUuid; + public void setVmInstanceUuid(java.lang.String vmInstanceUuid) { + this.vmInstanceUuid = vmInstanceUuid; + } + public java.lang.String getVmInstanceUuid() { + return this.vmInstanceUuid; + } + + public java.lang.String hostUuid; + public void setHostUuid(java.lang.String hostUuid) { + this.hostUuid = hostUuid; + } + public java.lang.String getHostUuid() { + return this.hostUuid; + } + + public java.lang.String type; + public void setType(java.lang.String type) { + this.type = type; + } + public java.lang.String getType() { + return this.type; + } + + public java.lang.String path; + public void setPath(java.lang.String path) { + this.path = path; + } + public java.lang.String getPath() { + return this.path; + } + + public java.lang.String lastSyncReason; + public void setLastSyncReason(java.lang.String lastSyncReason) { + this.lastSyncReason = lastSyncReason; + } + public java.lang.String getLastSyncReason() { + return this.lastSyncReason; + } + + public java.sql.Timestamp changeDate; + public void setChangeDate(java.sql.Timestamp changeDate) { + this.changeDate = changeDate; + } + public java.sql.Timestamp getChangeDate() { + return this.changeDate; + } + + public java.sql.Timestamp lastSyncDate; + public void setLastSyncDate(java.sql.Timestamp lastSyncDate) { + this.lastSyncDate = lastSyncDate; + } + public java.sql.Timestamp getLastSyncDate() { + return this.lastSyncDate; + } + + public java.sql.Timestamp createDate; + public void setCreateDate(java.sql.Timestamp createDate) { + this.createDate = createDate; + } + public java.sql.Timestamp getCreateDate() { + return this.createDate; + } + + public java.sql.Timestamp lastOpDate; + public void setLastOpDate(java.sql.Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } + public java.sql.Timestamp getLastOpDate() { + return this.lastOpDate; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStoragePathMaker.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStoragePathMaker.java index 7ef4b4dcd24..2458ee91d60 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStoragePathMaker.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStoragePathMaker.java @@ -43,6 +43,10 @@ public static String makeDataVolumeInstallPath(String volUuid) { return PathUtil.join("dataVolumes", "acct-" + getAccountUuidOfResource(volUuid), "vol-" + volUuid, volUuid + ".qcow2"); } + public static String makeNvRamVolumeInstallPath(String volUuid) { + return PathUtil.join("nvRam", "acct-" + getAccountUuidOfResource(volUuid), "vol-" + volUuid, volUuid + ".raw"); + } + public static String makeImageFromSnapshotWorkspacePath(String imageUuid) { return PathUtil.join("snapshotWorkspace", String.format("image-%s", imageUuid)); } diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotApiInterceptor.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotApiInterceptor.java index 33205212024..9cdcadf3fc0 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotApiInterceptor.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotApiInterceptor.java @@ -5,7 +5,6 @@ import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; -import org.zstack.core.db.SQL; import org.zstack.core.db.SimpleQuery; import org.zstack.core.db.SimpleQuery.Op; import org.zstack.core.errorcode.ErrorFacade; @@ -16,7 +15,6 @@ import org.zstack.header.message.APIMessage; import org.zstack.header.storage.snapshot.*; import org.zstack.header.storage.snapshot.group.*; -import org.zstack.header.vm.VmInstanceInventory; import org.zstack.header.vm.VmInstanceState; import org.zstack.header.vm.VmInstanceVO; import org.zstack.header.vm.VmInstanceVO_; @@ -29,7 +27,6 @@ import static org.zstack.storage.snapshot.VolumeSnapshotMessageRouter.getResourceIdToRouteMsg; import javax.persistence.Tuple; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; @@ -114,13 +111,18 @@ private void validate(APIRevertVmFromSnapshotGroupMsg msg) { } } - if (isWithMemoryForSnapshotGroup(group) - && Q.New(VmInstanceVO.class) - .eq(VmInstanceVO_.uuid, group.getVmInstanceUuid()) - .in(VmInstanceVO_.state, Arrays.asList(VmInstanceState.Running, VmInstanceState.Paused)) - .isExists()) { - throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_STORAGE_SNAPSHOT_10021, "Can not take memory snapshot, expected vm states are [%s, %s]", - VmInstanceState.Running.toString(), VmInstanceState.Paused.toString())); + if (isWithMemoryForSnapshotGroup(group)) { + VmInstanceState vmState = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.state) + .eq(VmInstanceVO_.uuid, group.getVmInstanceUuid()) + .findValue(); + if (!VolumeSnapshotConstant.ALLOW_TAKE_MEMORY_SNAPSHOTS_VM_STATES.contains(vmState)) { + throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_STORAGE_SNAPSHOT_10021, + "Can not revert VM with memory snapshot: unexpected VM state") + .withOpaque("vm.uuid", group.getVmInstanceUuid()) + .withOpaque("vm.state", vmState.toString()) + .withOpaque("expect.states", VolumeSnapshotConstant.ALLOW_TAKE_MEMORY_SNAPSHOTS_VM_STATES)); + } } msg.setVmInstanceUuid(group.getVmInstanceUuid()); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotGroupSystemTags.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotGroupSystemTags.java index b85d2f7eab4..7f931889782 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotGroupSystemTags.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotGroupSystemTags.java @@ -8,4 +8,8 @@ public class VolumeSnapshotGroupSystemTags { public static SystemTag VOLUME_SNAPSHOT_GROUP_CREATED_BY_SYSTEM = new SystemTag("CreatedBySystem", VolumeSnapshotGroupVO.class); + + // inherent system tag + public static SystemTag WITH_TPM = + new SystemTag("snapshotGroupWithTpm", VolumeSnapshotGroupVO.class); } diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java index 13a67af0970..3db27d21682 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java @@ -47,9 +47,13 @@ import org.zstack.header.storage.snapshot.VolumeSnapshotTree.SnapshotLeaf; import org.zstack.header.storage.snapshot.group.*; import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceVO; +import org.zstack.header.vm.VmInstanceConstant; import org.zstack.header.vm.VmInstanceState; import org.zstack.header.vm.VmInstanceVO; import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; import org.zstack.header.vm.devices.VmInstanceDeviceManager; import org.zstack.header.volume.*; import org.zstack.longjob.LongJobUtils; @@ -73,6 +77,8 @@ import static org.zstack.core.Platform.operr; import static org.zstack.core.progress.ProgressReportService.reportProgress; import static org.zstack.utils.CollectionDSL.e; +import static org.zstack.utils.CollectionDSL.list; +import static org.zstack.utils.CollectionUtils.transform; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; /** @@ -1377,6 +1383,7 @@ private void ungroupAfterDeleteSingleSnapshot(VolumeSnapshotInventory volumeSnap .eq(VolumeSnapshotGroupRefVO_.volumeSnapshotGroupUuid, volumeSnapshotInv.getGroupUuid()) .eq(VolumeSnapshotGroupRefVO_.snapshotDeleted, false).count(); if (count == 0) { + cleanVmHostBackupFilesForGroup(list(volumeSnapshotInv.getGroupUuid())); dbf.removeByPrimaryKey(volumeSnapshotInv.getGroupUuid(), VolumeSnapshotGroupVO.class); logger.debug(String.format("snapshot group[uuid:%s] all volume snapshot has been deleted, " + "delete snapshot group", volumeSnapshotInv.getGroupUuid())); @@ -2109,10 +2116,51 @@ private void ungroupAfterDeleted(List snapshots) { } groupUuids.forEach(groupUuid -> vidm.deleteArchiveVmInstanceDeviceAddressGroup(groupUuid)); + cleanVmHostBackupFilesForGroup(groupUuids); dbf.removeByPrimaryKeys(groupUuids, VolumeSnapshotGroupVO.class); } } + private void cleanVmHostBackupFilesForGroup(List groupUuids) { + // TODO: refactor this: VolumeSnapshotGroupVO should has its own cascade extensions! + // VolumeSnapshotGroupVO -> VmHostBackupFileVO + if (groupUuids.isEmpty()) { + return; + } + + List backupUuidList = Q.New(VmHostBackupFileVO.class) + .in(VmHostBackupFileVO_.resourceUuid, groupUuids) + .select(VmHostBackupFileVO_.uuid) + .listValues(); + if (!backupUuidList.isEmpty()) { + new While<>(backupUuidList).each((fileUuid, whileCompletion) -> { + VmHostBackupFileDeletionMsg deletionMsg = new VmHostBackupFileDeletionMsg(); + deletionMsg.setUuid(fileUuid); + deletionMsg.setForceDelete(true); + bus.makeLocalServiceId(deletionMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(deletionMsg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + whileCompletion.done(); + return; + } + whileCompletion.addError(reply.getError()); + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(null) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + logger.warn("failed to delete some VmHostBackupFiles:\n" + String.join("\n", + transform(errorCodeList.getCauses(), ErrorCode::getReadableDetails))); + } + } + }); + } + } + private List makeVolumeSnapshotBackupStorageDeletionMsg(List bsUuids) { List msgs = new ArrayList(); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/group/VolumeSnapshotGroupBase.java b/storage/src/main/java/org/zstack/storage/snapshot/group/VolumeSnapshotGroupBase.java index 6208b3f9bf4..687039c87be 100644 --- a/storage/src/main/java/org/zstack/storage/snapshot/group/VolumeSnapshotGroupBase.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/group/VolumeSnapshotGroupBase.java @@ -30,14 +30,26 @@ import org.zstack.header.storage.snapshot.VolumeSnapshotConstant; import org.zstack.header.storage.snapshot.VolumeSnapshotVO; import org.zstack.header.storage.snapshot.group.*; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; +import org.zstack.header.tpm.message.AddTpmMsg; +import org.zstack.header.tpm.message.DeleteTpmKeyBackupMsg; +import org.zstack.header.tpm.message.RestoreTpmEncryptionKeyMsg; +import org.zstack.header.tpm.message.RestoreTpmEncryptionKeyReply; +import org.zstack.header.tpm.message.TpmDeletionMsg; +import org.zstack.header.vm.additions.RestoreVmHostFileMsg; import org.zstack.header.vm.RestoreVmInstanceMsg; import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.additions.VmHostFileManager; +import org.zstack.header.vm.additions.VmHostFileSyncReason; import org.zstack.header.vm.devices.VmInstanceDeviceAddressArchiveVO; import org.zstack.header.vm.devices.VmInstanceDeviceManager; import org.zstack.header.volume.VolumeType; import org.zstack.header.volume.VolumeVO; import org.zstack.header.volume.VolumeVO_; import org.zstack.storage.snapshot.VolumeSnapshotGlobalConfig; +import org.zstack.storage.snapshot.VolumeSnapshotGroupSystemTags; +import org.zstack.utils.DebugUtils; import org.zstack.utils.TimeUtils; import org.zstack.utils.Utils; import org.zstack.utils.gson.JSONObjectUtil; @@ -47,6 +59,7 @@ import java.util.stream.Collectors; import static org.zstack.core.Platform.operr; +import static org.zstack.header.tpm.TpmConstants.SERVICE_ID; import static org.zstack.storage.snapshot.VolumeSnapshotMessageRouter.getResourceIdToRouteMsg; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; @@ -68,6 +81,8 @@ public class VolumeSnapshotGroupBase implements VolumeSnapshotGroup { private PluginRegistry pluginRgty; @Autowired private VmInstanceDeviceManager vidm; + @Autowired + private VmHostFileManager vmHostFileManager; public VolumeSnapshotGroupBase(VolumeSnapshotGroupVO self) { this.self = self; @@ -147,6 +162,7 @@ public String getSyncSignature() { public void run(SyncTaskChain chain) { APIUngroupVolumeSnapshotGroupEvent evt = new APIUngroupVolumeSnapshotGroupEvent(msg.getId()); dbf.remove(self); + vmHostFileManager.cleanVmHostBackupFile(msg.getUuid()); bus.publish(evt); chain.next(); } @@ -214,28 +230,41 @@ private void handle(DeleteVolumeSnapshotGroupInnerMsg msg) { logger.debug(String.format("skip snapshots not belong to origin vm[uuid:%s]", self.getVmInstanceUuid())); } - new While<>(snapshots).all((snapshot, compl) -> { - DeleteVolumeSnapshotMsg rmsg = new DeleteVolumeSnapshotMsg(); - rmsg.setSnapshotUuid(snapshot.getUuid()); - rmsg.setVolumeUuid(snapshot.getVolumeUuid()); - rmsg.setTreeUuid(snapshot.getTreeUuid()); - rmsg.setDeletionMode(msg.getDeletionMode()); - rmsg.setScope(msg.getScope()); - rmsg.setDirection(msg.getDirection()); - bus.makeTargetServiceIdByResourceUuid(rmsg, VolumeSnapshotConstant.SERVICE_ID, getResourceIdToRouteMsg(snapshot)); - bus.send(rmsg, new CloudBusCallBack(compl) { - @Override - public void run(MessageReply r) { - reply.addResult(new DeleteSnapshotGroupResult(rmsg.getSnapshotUuid(), rmsg.getVolumeUuid(), r.getError())); - compl.done(); - } - }); - }).run(new WhileDoneCompletion(msg) { - @Override - public void done(ErrorCodeList errorCodeList) { + SimpleFlowChain.of("delete-volume-snapshot-group") + .then("delete-volume-snapshots", (trigger) -> + new While<>(snapshots).step((snapshot, compl) -> { + DeleteVolumeSnapshotMsg rmsg = new DeleteVolumeSnapshotMsg(); + rmsg.setSnapshotUuid(snapshot.getUuid()); + rmsg.setVolumeUuid(snapshot.getVolumeUuid()); + rmsg.setTreeUuid(snapshot.getTreeUuid()); + rmsg.setDeletionMode(msg.getDeletionMode()); + rmsg.setScope(msg.getScope()); + rmsg.setDirection(msg.getDirection()); + bus.makeTargetServiceIdByResourceUuid(rmsg, VolumeSnapshotConstant.SERVICE_ID, getResourceIdToRouteMsg(snapshot)); + bus.send(rmsg, new CloudBusCallBack(compl) { + @Override + public void run(MessageReply r) { + reply.addResult(new DeleteSnapshotGroupResult(rmsg.getSnapshotUuid(), rmsg.getVolumeUuid(), r.getError())); + compl.done(); + } + }); + }, 5).run(new WhileDoneCompletion(msg) { + @Override + public void done(ErrorCodeList errorCodeList) { + trigger.next(); + } + })) + .then("delete-vm-host-backup-files", trigger -> { + vmHostFileManager.cleanVmHostBackupFile(self.getUuid()); + trigger.next(); + }) + .propagateExceptionTo(msg) + .done(() -> bus.reply(msg, reply)) + .error(errorCode -> { + reply.setError(errorCode); bus.reply(msg, reply); - } - }); + }) + .start(); } private void handle(APIRevertVmFromSnapshotGroupMsg msg) { @@ -363,47 +392,239 @@ private void handle(RevertVmFromSnapshotGroupInnerMsg msg) { return; } - VolumeSnapshotGroupVO newGroup = null; - if (VolumeSnapshotGlobalConfig.SNAPSHOT_BEFORE_REVERTVOLUME.value(Boolean.class)) { - newGroup = new VolumeSnapshotGroupVO(); - newGroup.setUuid(Platform.getUuid()); - newGroup.setName(String.format("revert-vm-point-%s-%s", vmUuid, TimeUtils.getCurrentTimeStamp("yyyyMMddHHmmss"))); - newGroup.setDescription(String.format("save snapshot for revert vm [uuid:%s]", vmUuid)); - newGroup.setSnapshotCount(snapshots.size()); - newGroup.setVmInstanceUuid(vmUuid); - newGroup.setAccountUuid(msg.getSession().getAccountUuid()); - dbf.persist(newGroup); + class Context { + VolumeSnapshotGroupVO newGroup; + boolean snapshotGroupHasTpm; + String tpmUuid, newCreateTpmUuid; + String tpmKeyBackupUuid; } + Context context = new Context(); + context.snapshotGroupHasTpm = VolumeSnapshotGroupSystemTags.WITH_TPM.hasTag(msg.getUuid()); + context.tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vmUuid) + .select(TpmVO_.uuid) + .findValue(); + + SimpleFlowChain.of("revert-vm-from-snapshot-group-inner") + .then(Flow.of("persist-before-revert-snapshot-in-db") + .runIf(data -> VolumeSnapshotGlobalConfig.SNAPSHOT_BEFORE_REVERTVOLUME.value(Boolean.class)) + .handle(trigger -> { + context.newGroup = new VolumeSnapshotGroupVO(); + context.newGroup.setUuid(Platform.getUuid()); + context.newGroup.setName(String.format("revert-vm-point-%s-%s", vmUuid, TimeUtils.getCurrentTimeStamp("yyyyMMddHHmmss"))); + context.newGroup.setDescription(String.format("save snapshot for revert vm [uuid:%s]", vmUuid)); + context.newGroup.setSnapshotCount(snapshots.size()); + context.newGroup.setVmInstanceUuid(vmUuid); + context.newGroup.setAccountUuid(msg.getSession().getAccountUuid()); + dbf.persist(context.newGroup); + trigger.next(); + }) + .build()) + .then(Flow.of("create-tpm-if-needed") + .runIf(data -> context.snapshotGroupHasTpm && context.tpmUuid == null) + .handle(trigger -> { + AddTpmMsg addTpmMsg = new AddTpmMsg(); + addTpmMsg.setResourceUuidKeyFrom(msg.getUuid()); + addTpmMsg.setVmInstanceUuid(vmUuid); + addTpmMsg.setTpmUuid(context.newCreateTpmUuid = Platform.getUuid()); + bus.makeTargetServiceIdByResourceUuid(addTpmMsg, SERVICE_ID, addTpmMsg.getTpmUuid()); + bus.send(addTpmMsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + logger.debug(String.format("create Tpm[uuid:%s] for VM[uuid:%s]", + addTpmMsg.getTpmUuid(), addTpmMsg.getVmInstanceUuid())); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + }) + .rollback(trigger -> { + TpmDeletionMsg deletionMsg = new TpmDeletionMsg(); + deletionMsg.setTpmUuid(context.newCreateTpmUuid); + deletionMsg.setVmInstanceUuid(vmUuid); + deletionMsg.setForceDelete(true); + bus.makeTargetServiceIdByResourceUuid(deletionMsg, SERVICE_ID, deletionMsg.getTpmUuid()); + bus.send(deletionMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.debug(String.format("failed to deleted Tpm[uuid:%s] from VM[uuid:%s] but still continue", + deletionMsg.getTpmUuid(), deletionMsg.getVmInstanceUuid())); + } + trigger.rollback(); + } + }); + }) + .build()) + .then(Flow.of("restore-tpm") + .runIf(data -> context.snapshotGroupHasTpm && context.tpmUuid != null) + .handle(trigger -> { + RestoreTpmEncryptionKeyMsg restoreMsg = new RestoreTpmEncryptionKeyMsg(); + restoreMsg.setSrcResourceUuid(msg.getUuid()); + restoreMsg.setDstResourceUuid(context.tpmUuid); + restoreMsg.setBackupCurrentKey(true); + bus.makeTargetServiceIdByResourceUuid(restoreMsg, SERVICE_ID, context.tpmUuid); + bus.send(restoreMsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + if (reply instanceof RestoreTpmEncryptionKeyReply) { + context.tpmKeyBackupUuid = ((RestoreTpmEncryptionKeyReply) reply).getTpmKeyBackupUuid(); + } + logger.debug(String.format( + "restore resource key of Tpm[uuid:%s] for VM[uuid:%s] for snapshotGroup[uuid:%s]", + context.tpmUuid, vmUuid, msg.getUuid())); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + }) + .rollback(trigger -> { + if (context.tpmKeyBackupUuid == null) { + trigger.rollback(); + return; + } + RestoreTpmEncryptionKeyMsg rollbackMsg = new RestoreTpmEncryptionKeyMsg(); + rollbackMsg.setBackupCurrentKey(false); + rollbackMsg.setSrcResourceUuid(context.tpmKeyBackupUuid); + rollbackMsg.setDstResourceUuid(context.tpmUuid); + bus.makeTargetServiceIdByResourceUuid(rollbackMsg, SERVICE_ID, context.tpmUuid); + bus.send(rollbackMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + logger.debug(String.format( + "failed to rollback TPM encryption key from TpmKeyBackupVO[uuid:%s] to Tpm[uuid:%s]", + context.tpmKeyBackupUuid, context.tpmUuid)); + } + DeleteTpmKeyBackupMsg delMsg = new DeleteTpmKeyBackupMsg(); + delMsg.setTpmUuid(context.tpmUuid); + delMsg.setTpmKeyBackupUuid(context.tpmKeyBackupUuid); + bus.makeTargetServiceIdByResourceUuid(delMsg, SERVICE_ID, context.tpmUuid); + bus.send(delMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply2) { + if (!reply2.isSuccess()) { + logger.debug(String.format( + "failed to delete TpmKeyBackupVO[uuid:%s] after TPM key rollback", + context.tpmKeyBackupUuid)); + } + context.tpmKeyBackupUuid = null; + trigger.rollback(); + } + }); + } + }); + }) + .build()) + .then(Flow.of("remove-tpm-if-needed") + .runIf(data -> !context.snapshotGroupHasTpm && context.tpmUuid != null) + .handle(trigger -> { + TpmDeletionMsg deletionMsg = new TpmDeletionMsg(); + deletionMsg.setTpmUuid(context.tpmUuid); + deletionMsg.setVmInstanceUuid(vmUuid); + deletionMsg.setForceDelete(true); + bus.makeTargetServiceIdByResourceUuid(deletionMsg, SERVICE_ID, deletionMsg.getTpmUuid()); + bus.send(deletionMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + logger.debug(String.format("deleted Tpm[uuid:%s] from VM[uuid:%s]", + deletionMsg.getTpmUuid(), deletionMsg.getVmInstanceUuid())); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + }) + // TODO: It should has rollback + .build()) + .then(Flow.of("restore-vm-host-file") + .handle(trigger -> { + RestoreVmHostFileMsg restoreMsg = new RestoreVmHostFileMsg(); + restoreMsg.setVmInstanceUuid(vmUuid); + restoreMsg.setSnapshotGroupUuid(self.getUuid()); + restoreMsg.setSyncReason(VmHostFileSyncReason.RevertSnapshot.reason()); + bus.makeLocalServiceId(restoreMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(restoreMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + }) + .build()) + .then(Flow.of("revert-every-volumes") + .handle(trigger -> { + final String finalNewGroupUuid = context.newGroup == null ? null : context.newGroup.getUuid(); + Integer concurrencyNum = VolumeSnapshotGlobalConfig.SNAPSHOT_GROUP_REVERT_CONCURRENCY.value(Integer.class); + new While<>(snapshots).step((snapshot, compl) -> { + if (Q.New(VolumeVO.class).eq(VolumeVO_.uuid, snapshot.getVolumeUuid()).eq(VolumeVO_.type, VolumeType.Memory).isExists()) { + compl.done(); + return; + } - final String finalNewGroupUuid = newGroup == null ? null : newGroup.getUuid(); - Integer concurrencyNum = VolumeSnapshotGlobalConfig.SNAPSHOT_GROUP_REVERT_CONCURRENCY.value(Integer.class); - new While<>(snapshots).step((snapshot, compl) -> { - if (Q.New(VolumeVO.class).eq(VolumeVO_.uuid, snapshot.getVolumeUuid()).eq(VolumeVO_.type, VolumeType.Memory).isExists()) { - compl.done(); - return; - } - - RevertVolumeFromSnapshotGroupMsg rmsg = new RevertVolumeFromSnapshotGroupMsg(); - rmsg.setSnapshotUuid(snapshot.getUuid()); - rmsg.setVolumeUuid(snapshot.getVolumeUuid()); - rmsg.setTreeUuid(snapshot.getTreeUuid()); - rmsg.setSession(msg.getSession()); - rmsg.setNewSnapshotGroupUuid(finalNewGroupUuid); - - bus.makeTargetServiceIdByResourceUuid(rmsg, VolumeSnapshotConstant.SERVICE_ID, getResourceIdToRouteMsg(snapshot)); - bus.send(rmsg, new CloudBusCallBack(compl) { - @Override - public void run(MessageReply r) { - reply.addResult(new RevertSnapshotGroupResult(rmsg.getSnapshotUuid(), rmsg.getVolumeUuid(), r.getError())); - compl.done(); - } - }); - }, concurrencyNum).run(new WhileDoneCompletion(msg) { - @Override - public void done(ErrorCodeList errorCodeList) { + RevertVolumeFromSnapshotGroupMsg rmsg = new RevertVolumeFromSnapshotGroupMsg(); + rmsg.setSnapshotUuid(snapshot.getUuid()); + rmsg.setVolumeUuid(snapshot.getVolumeUuid()); + rmsg.setTreeUuid(snapshot.getTreeUuid()); + rmsg.setSession(msg.getSession()); + rmsg.setNewSnapshotGroupUuid(finalNewGroupUuid); + + bus.makeTargetServiceIdByResourceUuid(rmsg, VolumeSnapshotConstant.SERVICE_ID, getResourceIdToRouteMsg(snapshot)); + bus.send(rmsg, new CloudBusCallBack(compl) { + @Override + public void run(MessageReply r) { + reply.addResult(new RevertSnapshotGroupResult(rmsg.getSnapshotUuid(), rmsg.getVolumeUuid(), r.getError())); + compl.done(); + } + }); + }, concurrencyNum).run(new WhileDoneCompletion(msg) { + @Override + public void done(ErrorCodeList errorCodeList) { + DebugUtils.Assert(errorCodeList.getCauses().isEmpty(), "no errorCode expected"); + trigger.next(); + } + }); + }) + .build()) + .then(Flow.of("delete-tpm-key-backup") + .runIf(data -> context.tpmKeyBackupUuid != null) + .handle(trigger -> { + DeleteTpmKeyBackupMsg delMsg = new DeleteTpmKeyBackupMsg(); + delMsg.setTpmUuid(context.tpmUuid); + delMsg.setTpmKeyBackupUuid(context.tpmKeyBackupUuid); + bus.makeTargetServiceIdByResourceUuid(delMsg, SERVICE_ID, context.tpmUuid); + bus.send(delMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply r) { + if (r.isSuccess()) { + context.tpmKeyBackupUuid = null; + trigger.next(); + } else { + trigger.fail(r.getError()); + } + } + }); + }) + .build()) + .propagateExceptionTo(msg) + .done(() -> bus.reply(msg, reply)) + .error(errorCode -> { + reply.setError(errorCode); bus.reply(msg, reply); - } - }); + }) + .start(); } public List getSnapshots() { diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java index 2e87e6b41e8..8f5b88d8e8d 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java @@ -38,16 +38,24 @@ import org.zstack.header.storage.snapshot.group.*; import org.zstack.header.tag.SystemTagVO; import org.zstack.header.tag.SystemTagVO_; +import org.zstack.header.tpm.entity.TpmVO; +import org.zstack.header.tpm.entity.TpmVO_; import org.zstack.header.vm.*; +import org.zstack.header.tpm.TpmConstants; +import org.zstack.header.vm.additions.VmHostBackupFileDeletionMsg; +import org.zstack.header.vm.additions.VmHostBackupFileVO; +import org.zstack.header.vm.additions.VmHostBackupFileVO_; import org.zstack.header.vm.devices.VmInstanceDeviceManager; import org.zstack.header.volume.*; import org.zstack.header.volume.VolumeConstant.Capability; import org.zstack.header.volume.VolumeDeletionPolicyManager.VolumeDeletionPolicy; import org.zstack.identity.AccountManager; +import org.zstack.header.tpm.message.BackupTpmEncryptionKeyMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageReply; import org.zstack.storage.primary.PrimaryStorageDeleteBitGC; import org.zstack.storage.primary.PrimaryStorageGlobalConfig; +import org.zstack.storage.snapshot.VolumeSnapshotGroupSystemTags; import org.zstack.storage.snapshot.group.VolumeSnapshotGroupOperationValidator; import org.zstack.storage.snapshot.reference.VolumeSnapshotReferenceUtils; import org.zstack.tag.SystemTagCreator; @@ -63,12 +71,14 @@ import javax.persistence.TypedQuery; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static org.zstack.core.Platform.*; import static org.zstack.storage.volume.VolumeSystemTags.VOLUME_PROVISIONING_STRATEGY; import static org.zstack.storage.volume.VolumeSystemTags.VOLUME_PROVISIONING_STRATEGY_TOKEN; import static org.zstack.utils.CollectionDSL.*; +import static org.zstack.utils.CollectionUtils.transform; import static org.zstack.utils.clouderrorcode.CloudOperationsErrorCode.*; /** @@ -817,7 +827,8 @@ public void fail(ErrorCode errorCode) { } else if (msg instanceof InstantiateMemoryVolumeMsg) { instantiateMemoryVolume(msg, trigger); } else { - instantiateDataVolume(msg, trigger); + // include: data volume, NvRam, TpmState + instantiateOtherVolume(msg, trigger); } } } @@ -859,7 +870,7 @@ private void instantiateRootVolume(InstantiateRootVolumeMsg msg, FlowTrigger tri doInstantiateVolume(imsg, trigger); } - private void instantiateDataVolume(InstantiateVolumeMsg msg, FlowTrigger trigger) { + private void instantiateOtherVolume(InstantiateVolumeMsg msg, FlowTrigger trigger) { InstantiateVolumeOnPrimaryStorageMsg imsg = new InstantiateVolumeOnPrimaryStorageMsg(); prepareMsg(msg, imsg); doInstantiateVolume(imsg, trigger); @@ -3345,12 +3356,14 @@ private void createSnapshotGroup(CreateVolumeSnapshotGroupMessage msg, ReturnVal CreateVolumesSnapshotMsg cmsg = new CreateVolumesSnapshotMsg(); List volumesSnapshotsJobs = new ArrayList<>(); cmsg.setAccountUuid(msg.getSession().getAccountUuid()); + cmsg.setBackupHostFileIfNeeded(true); VmInstanceInventory vm = msg.getVmInstance(); Map vols = vm.getAllVolumes().stream() .filter(it -> it.isDisk() || msg.getConsistentType() == ConsistentType.Application && it.getType().equals(VolumeType.Memory.toString())) .collect(Collectors.toMap(VolumeInventory::getUuid, it -> it)); + boolean withMemory = msg.getConsistentType() == ConsistentType.Application; for (VolumeInventory vol : vols.values()) { CreateVolumesSnapshotsJobStruct volumesSnapshotsJob = new CreateVolumesSnapshotsJobStruct(); @@ -3365,55 +3378,157 @@ private void createSnapshotGroup(CreateVolumeSnapshotGroupMessage msg, ReturnVal cmsg.setVolumeSnapshotJobs(volumesSnapshotsJobs); cmsg.setConsistentType(msg.getConsistentType()); - bus.makeTargetServiceIdByResourceUuid(cmsg, VolumeConstant.SERVICE_ID, msg.getRootVolumeUuid()); - bus.send(cmsg, new CloudBusCallBack(completion) { - @Override - public void run(MessageReply reply) { - if (!reply.isSuccess()) { - completion.fail(reply.getError()); - return; - } + List inventories = new ArrayList<>(); + List hostBackupFileUuidList = new ArrayList<>(); + AtomicReference groupRef = new AtomicReference<>(null); + String resourceUuid = msg.getResourceUuid() == null ? getUuid() : msg.getResourceUuid(); + SimpleFlowChain.of("create-snapshot-group") + .then("handle-create-snapshot-group-message", trigger -> { + bus.makeTargetServiceIdByResourceUuid(cmsg, VolumeConstant.SERVICE_ID, msg.getRootVolumeUuid()); + bus.send(cmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + trigger.fail(reply.getError()); + return; + } + CreateVolumesSnapshotReply r = reply.castReply(); + inventories.addAll(r.getInventories()); + if (!CollectionUtils.isEmpty(r.getHostBackupFileUuidList())) { + hostBackupFileUuidList.addAll(r.getHostBackupFileUuidList()); + } + trigger.next(); + } + }); + }) + .then(Flow.of("persist-snapshot-group") + .handle(trigger -> { + List refs = new ArrayList<>(); + VolumeSnapshotGroupVO group = new VolumeSnapshotGroupVO(); + group.setUuid(resourceUuid); + group.setSnapshotCount(cmsg.getVolumeSnapshotJobs().size()); + group.setName(msg.getName()); + group.setDescription(msg.getDescription()); + group.setVmInstanceUuid(vm.getUuid()); + group.setAccountUuid(msg.getSession().getAccountUuid()); + for (VolumeSnapshotInventory inv : inventories) { + VolumeSnapshotGroupRefVO ref = new VolumeSnapshotGroupRefVO(); + ref.setVolumeUuid(inv.getVolumeUuid()); + ref.setVolumeName(vols.get(inv.getVolumeUuid()).getName()); + ref.setVolumeType(inv.getVolumeType()); + ref.setVolumeSnapshotGroupUuid(group.getUuid()); + ref.setVolumeSnapshotUuid(inv.getUuid()); + ref.setVolumeSnapshotName(inv.getName()); + ref.setVolumeSnapshotInstallPath(inv.getPrimaryStorageInstallPath()); + ref.setDeviceId(vols.get(inv.getVolumeUuid()).getDeviceId()); + ref.setVolumeLastAttachDate(vols.get(inv.getVolumeUuid()).getLastAttachDate()); + refs.add(ref); + } + + new SQLBatch() { + @Override + protected void scripts() { + databaseFacade.persist(group); + databaseFacade.persistCollection(refs); + } + }.execute(); - CreateVolumesSnapshotReply r = reply.castReply(); - VolumeSnapshotGroupVO group = createGroup(r); - logger.debug(String.format("created volume snapshot group[uuid:%s] for vm[uuid:%s]", - group.getUuid(), vm.getUuid())); - completion.success(VolumeSnapshotGroupInventory.valueOf(dbf.reload(group))); - } + groupRef.set(group); + trigger.next(); + }) + .rollback(trigger -> { + SQL.New(VolumeSnapshotGroupVO.class) + .eq(VolumeSnapshotGroupVO_.uuid, resourceUuid) + .delete(); + // VolumeSnapshotGroupRefVO delete in cascade + trigger.rollback(); + }) + .build()) + .then(Flow.of("persist-vm-host-backup-file") + .handle(trigger -> { + if (!CollectionUtils.isEmpty(hostBackupFileUuidList)) { + SQL.New(VmHostBackupFileVO.class) + .in(VmHostBackupFileVO_.uuid, hostBackupFileUuidList) + .set(VmHostBackupFileVO_.resourceUuid, resourceUuid) + .update(); + } + trigger.next(); + }) + .rollback(trigger -> { + if (CollectionUtils.isEmpty(hostBackupFileUuidList)) { + trigger.rollback(); + return; + } - private VolumeSnapshotGroupVO createGroup(CreateVolumesSnapshotReply r) { - List refs = new ArrayList<>(); - VolumeSnapshotGroupVO group = new VolumeSnapshotGroupVO(); - if (msg.getResourceUuid() != null) { - group.setUuid(msg.getResourceUuid()); - } else { - group.setUuid(getUuid()); - } - group.setSnapshotCount(cmsg.getVolumeSnapshotJobs().size()); - group.setName(msg.getName()); - group.setDescription(msg.getDescription()); - group.setVmInstanceUuid(vm.getUuid()); - group.setAccountUuid(msg.getSession().getAccountUuid()); - for (VolumeSnapshotInventory inv : r.getInventories()) { - VolumeSnapshotGroupRefVO ref = new VolumeSnapshotGroupRefVO(); - ref.setVolumeUuid(inv.getVolumeUuid()); - ref.setVolumeName(vols.get(inv.getVolumeUuid()).getName()); - ref.setVolumeType(inv.getVolumeType()); - ref.setVolumeSnapshotGroupUuid(group.getUuid()); - ref.setVolumeSnapshotUuid(inv.getUuid()); - ref.setVolumeSnapshotName(inv.getName()); - ref.setVolumeSnapshotInstallPath(inv.getPrimaryStorageInstallPath()); - ref.setDeviceId(vols.get(inv.getVolumeUuid()).getDeviceId()); - ref.setVolumeLastAttachDate(vols.get(inv.getVolumeUuid()).getLastAttachDate()); - refs.add(ref); - } + new While<>(hostBackupFileUuidList).each((uuid, whileCompletion) -> { + VmHostBackupFileDeletionMsg deletionMsg = new VmHostBackupFileDeletionMsg(); + deletionMsg.setUuid(uuid); + deletionMsg.setForceDelete(true); + bus.makeLocalServiceId(deletionMsg, VmInstanceConstant.SECURE_BOOT_SERVICE_ID); + bus.send(deletionMsg, new CloudBusCallBack(whileCompletion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + whileCompletion.done(); + return; + } + whileCompletion.addError(reply.getError()); + whileCompletion.done(); + } + }); + }).run(new WhileDoneCompletion(trigger) { + @Override + public void done(ErrorCodeList errorCodeList) { + if (!errorCodeList.getCauses().isEmpty()) { + logger.warn("failed to delete some VmHostBackupFiles:\n" + String.join("\n", + transform(errorCodeList.getCauses(), ErrorCode::getReadableDetails))); + } + trigger.rollback(); + } + }); + }) + .build()) + .then(Flow.of("backup-encrypted-resource-key-ref-for-snapshot-group") + .skipIf(data -> CollectionUtils.isEmpty(hostBackupFileUuidList) || !withMemory) + .handle(trigger -> { + String tpmUuid = Q.New(TpmVO.class) + .eq(TpmVO_.vmInstanceUuid, vm.getUuid()) + .select(TpmVO_.uuid) + .findValue(); + if (tpmUuid == null) { + trigger.next(); + return; + } - dbf.persist(group); - dbf.persistCollection(refs); - return group; - } - }); + tagMgr.createInherentSystemTag(resourceUuid, + VolumeSnapshotGroupSystemTags.WITH_TPM.getTagFormat(), + VolumeSnapshotGroupVO.class.getSimpleName()); + + BackupTpmEncryptionKeyMsg backupMsg = new BackupTpmEncryptionKeyMsg(); + backupMsg.setSrcResourceUuid(tpmUuid); + backupMsg.setDstResourceUuid(resourceUuid); + bus.makeLocalServiceId(backupMsg, TpmConstants.SERVICE_ID); + bus.send(backupMsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + trigger.fail(reply.getError()); + return; + } + trigger.next(); + } + }); + }) + .build()) + .propagateExceptionTo(completion) + .done(() -> { + logger.debug(String.format("created volume snapshot group[uuid:%s] for vm[uuid:%s]", + groupRef.get().getUuid(), vm.getUuid())); + completion.success(VolumeSnapshotGroupInventory.valueOf(dbf.reload(groupRef.get()))); + }) + .error(completion::fail) + .start(); } private void handle(APIFlattenVolumeMsg msg) { @@ -3638,7 +3753,8 @@ private void handle(APIChangeVolumeProtocolMsg msg) { .param("uuid", msg.getVolumeUuid()) .execute(); if (updated == 0) { - evt.setError(operr("volume[uuid:%s] no longer exists, cannot change its protocol", msg.getVolumeUuid())); + evt.setError(operr(ORG_ZSTACK_STORAGE_VOLUME_10101, + "volume[uuid:%s] no longer exists, cannot change its protocol", msg.getVolumeUuid())); bus.publish(evt); return; } diff --git a/test/pom.xml b/test/pom.xml index 6505ec9324c..f90034ca817 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -6,12 +6,12 @@ ${skipJacoco} 4.0.0 - - zstack - org.zstack - 5.5.0 - .. - + + zstack + org.zstack + 5.5.0 + .. + test \n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def queryKeyProvider(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.api.QueryKeyProviderAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.api.QueryKeyProviderAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def rekeyKeyProviderRefs(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.api.RekeyKeyProviderRefsAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.api.RekeyKeyProviderRefsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def createKms(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.CreateKmsAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.CreateKmsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def deleteKms(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.DeleteKmsAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.DeleteKmsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def getKmsServerCertFromKms(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.GetKmsServerCertFromKmsAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.GetKmsServerCertFromKmsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def queryKms(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.QueryKmsAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.QueryKmsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def updateKms(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.UpdateKmsAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.UpdateKmsAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def uploadKmsClientCsr(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.UploadKmsClientCsrAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.UploadKmsClientCsrAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def uploadKmsClientIdentity(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.UploadKmsClientIdentityAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.UploadKmsClientIdentityAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def uploadKmsClientSignedCert(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.UploadKmsClientSignedCertAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.UploadKmsClientSignedCertAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def uploadKmsServerCert(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.kms.api.UploadKmsServerCertAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.kms.api.UploadKmsServerCertAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def backupNkp(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.BackupNkpAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.BackupNkpAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def createNkp(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.CreateNkpAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.CreateNkpAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def deleteNkp(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.DeleteNkpAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.DeleteNkpAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def parseNkpRestore(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.ParseNkpRestoreAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.ParseNkpRestoreAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def queryNkp(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.QueryNkpAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.QueryNkpAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def restoreNkp(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.RestoreNkpAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.RestoreNkpAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() if (System.getProperty("apipath") != null) { @@ -52820,6 +53285,31 @@ abstract class ApiHelper { } + def updateNkp(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.keyprovider.nkp.api.UpdateNkpAction.class) Closure c) { + def a = new org.zstack.sdk.keyprovider.nkp.api.UpdateNkpAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } def getLicenseAuthorizedCapacity(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.license.api.server.GetLicenseAuthorizedCapacityAction.class) Closure c) { def a = new org.zstack.sdk.license.api.server.GetLicenseAuthorizedCapacityAction() a.sessionId = Test.currentEnvSpec?.session?.uuid @@ -58517,4 +59007,141 @@ abstract class ApiHelper { } + def addTpm(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.tpm.api.AddTpmAction.class) Closure c) { + def a = new org.zstack.sdk.tpm.api.AddTpmAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def getTpmCapability(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.tpm.api.GetTpmCapabilityAction.class) Closure c) { + def a = new org.zstack.sdk.tpm.api.GetTpmCapabilityAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def queryTpm(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.tpm.api.QueryTpmAction.class) Closure c) { + def a = new org.zstack.sdk.tpm.api.QueryTpmAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + a.conditions = a.conditions.collect { it.toString() } + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def removeTpm(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.tpm.api.RemoveTpmAction.class) Closure c) { + def a = new org.zstack.sdk.tpm.api.RemoveTpmAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + + def updateTpm(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.tpm.api.UpdateTpmAction.class) Closure c) { + def a = new org.zstack.sdk.tpm.api.UpdateTpmAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + } diff --git a/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy b/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy index 1aeb890345b..7193e74546a 100755 --- a/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy +++ b/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy @@ -1,7 +1,8 @@ package org.zstack.testlib -import org.springframework.http.HttpEntity -import org.zstack.core.db.Q +import org.springframework.http.HttpEntity +import org.zstack.core.Platform +import org.zstack.core.db.Q import org.zstack.core.db.SQL import org.zstack.core.db.SQLBatch import org.zstack.header.Constants @@ -12,6 +13,7 @@ import org.zstack.header.storage.snapshot.TakeSnapshotsOnKvmResultStruct import org.zstack.header.vm.VmInstanceState import org.zstack.header.vm.VmInstanceVO import org.zstack.header.vm.VmInstanceVO_ +import org.zstack.header.vm.additions.VmHostFileContentFormat import org.zstack.header.vm.devices.DeviceAddress import org.zstack.header.vm.devices.VirtualDeviceInfo import org.zstack.header.volume.VolumeInventory @@ -35,9 +37,11 @@ import static org.zstack.kvm.KVMAgentCommands.* /** * Created by xing5 on 2017/6/6. */ -class KVMSimulator implements Simulator { - static ConcurrentHashMap connectCmdConcurrentHashMap = new ConcurrentHashMap<>() - private static Map takeSnapshotBackends = [:] +class KVMSimulator implements Simulator { + static ConcurrentHashMap connectCmdConcurrentHashMap = new ConcurrentHashMap<>() + static ConcurrentHashMap ensureSecretUuidCache = new ConcurrentHashMap<>() + static final String MOCK_ENVELOPE_PUBLIC_KEY_BASE64 = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" + private static Map takeSnapshotBackends = [:] static { BeanUtils.reflections.getSubTypesOf(VFSPrimaryStorageTakeSnapshotBackend.class).each { clz -> @@ -298,23 +302,79 @@ class KVMSimulator implements Simulator { return rsp } - spec.simulator(KVMConstant.KVM_CONNECT_PATH) { HttpEntity e -> - Spec.checkHttpCallType(e, true) - KVMAgentCommands.ConnectCmd cmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.ConnectCmd.class) + spec.simulator(KVMConstant.KVM_CONNECT_PATH) { HttpEntity e -> + Spec.checkHttpCallType(e, true) + KVMAgentCommands.ConnectCmd cmd = JSONObjectUtil.toObject(e.body, KVMAgentCommands.ConnectCmd.class) connectCmdConcurrentHashMap.put(cmd.hostUuid, cmd) def rsp = new KVMAgentCommands.ConnectResponse() rsp.success = true rsp.libvirtVersion = "1.0.0" - rsp.qemuVersion = "1.3.0" - return rsp - } - - spec.simulator(KVMConstant.KVM_ECHO_PATH) { HttpEntity e -> - Spec.checkHttpCallType(e, true) - return [:] - } + rsp.qemuVersion = "1.3.0" + return rsp + } + + spec.simulator(KVMConstant.KVM_CREATE_ENVELOPE_KEY_PATH) { + def rsp = new CreatePublicKeyResponse() + rsp.success = true + return rsp + } + + spec.simulator(KVMConstant.KVM_GET_ENVELOPE_KEY_PATH) { + def rsp = new GetPublicKeyResponse() + rsp.success = true + rsp.publicKey = MOCK_ENVELOPE_PUBLIC_KEY_BASE64 + return rsp + } + + spec.simulator(KVMConstant.KVM_VERIFY_ENVELOPE_KEY_PATH) { + def rsp = new VerifyPublicKeyResponse() + rsp.success = true + return rsp + } + + spec.simulator(KVMConstant.KVM_ROTATE_ENVELOPE_KEY_PATH) { + def rsp = new RotatePublicKeyResponse() + rsp.success = true + return rsp + } + + spec.simulator(KVMConstant.KVM_ENSURE_SECRET_PATH) { HttpEntity e -> + String hostUuid = e.getHeaders().getFirst(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID) + SecretHostDefineCmd cmd = JSONObjectUtil.toObject(e.body, SecretHostDefineCmd.class) + String cacheKey = buildHostSecretCacheKey(hostUuid, cmd?.vmUuid, cmd?.purpose, cmd?.keyVersion, cmd?.usageInstance) + def rsp = new SecretHostDefineResponse() + rsp.secretUuid = ensureSecretUuidCache.computeIfAbsent(cacheKey) { Platform.uuid } + return rsp + } + + spec.simulator(KVMConstant.KVM_GET_SECRET_PATH) { HttpEntity e -> + String hostUuid = e.getHeaders().getFirst(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID) + SecretHostGetCmd cmd = JSONObjectUtil.toObject(e.body, SecretHostGetCmd.class) + String cacheKey = buildHostSecretCacheKey(hostUuid, cmd?.vmUuid, cmd?.purpose, cmd?.keyVersion, cmd?.usageInstance) + def rsp = new SecretHostGetResponse() + String secretUuid = ensureSecretUuidCache.get(cacheKey) + if (secretUuid == null) { + rsp.setError("KEY_AGENT_SECRET_NOT_FOUND") + } else { + rsp.secretUuid = secretUuid + } + return rsp + } + + spec.simulator(KVMConstant.KVM_DELETE_SECRET_PATH) { HttpEntity e -> + String hostUuid = e.getHeaders().getFirst(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID) + SecretHostDeleteCmd cmd = JSONObjectUtil.toObject(e.body, SecretHostDeleteCmd.class) + String cacheKey = buildHostSecretCacheKey(hostUuid, cmd?.vmUuid, cmd?.purpose, cmd?.keyVersion, cmd?.usageInstance) + ensureSecretUuidCache.remove(cacheKey) + return new SecretHostDeleteResponse() + } + + spec.simulator(KVMConstant.KVM_ECHO_PATH) { HttpEntity e -> + Spec.checkHttpCallType(e, true) + return [:] + } spec.simulator(KVMConstant.KVM_DETACH_VOLUME) { return new KVMAgentCommands.DetachDataVolumeResponse() @@ -519,6 +579,7 @@ class KVMSimulator implements Simulator { rsp.virtualizerInfo.uuid = cmd.vmInstanceUuid rsp.virtualizerInfo.virtualizer = "qemu-kvm" rsp.virtualizerInfo.version = "4.2.0-632.g6a6222b.el7" + rsp.edkRpm = "edk2-ovmf-20220126gitbb1bba3d77-3.el8.noarch" return rsp } @@ -703,5 +764,52 @@ class KVMSimulator implements Simulator { spec.simulator(KVMConstant.HA_NETWORK_GROUP_SYNC_PATH) { return new KVMAgentCommands.AgentResponse() } - } -} + + spec.simulator(KVMConstant.READ_VM_HOST_FILE_PATH) { HttpEntity e -> + def cmd = JSONObjectUtil.toObject(e.body, ReadVmHostFileContentCmd) + + def rsp = new ReadVmHostFileContentResponse() + for (final def param in cmd.hostFiles) { + def to = new VmHostFileTO() + to.path = param.path + to.type = param.type + to.fileFormat = VmHostFileContentFormat.Raw.toString() + to.contentBase64 = "dGVzdA==" + rsp.hostFiles.add(to) + } + return rsp + } + + spec.simulator(KVMConstant.WRITE_VM_HOST_FILE_PATH) { HttpEntity e -> + return new WriteVmHostFileContentResponse() + } + + spec.simulator(KVMConstant.BACKUP_VM_HOST_FILE_PATH) { HttpEntity e -> + return new BackupVmHostFileResponse() + } + } + + /** Resets simulated host vTPM secret entries for getSecret / ensureSecret / deleteSecret. */ + static void resetSimulatedHostSecretCache() { + ensureSecretUuidCache.clear() + } + + static void putSimulatedHostSecretForTest(String hostUuid, String vmUuid, String purpose, + Integer keyVersion, String secretUuid) { + ensureSecretUuidCache.put(buildHostSecretCacheKey(hostUuid, vmUuid, purpose, keyVersion, KVMConstant.HOST_SECRET_USAGE_INSTANCE_VTPM), secretUuid) + } + + static String getSimulatedHostSecretForTest(String hostUuid, String vmUuid, String purpose, Integer keyVersion) { + return ensureSecretUuidCache.get(buildHostSecretCacheKey(hostUuid, vmUuid, purpose, keyVersion, KVMConstant.HOST_SECRET_USAGE_INSTANCE_VTPM)) + } + + private static String buildHostSecretCacheKey(String hostUuid, String vmUuid, String purpose, Integer keyVersion, String usageInstance) { + String usage = usageInstance ?: KVMConstant.HOST_SECRET_USAGE_INSTANCE_VTPM + return String.format("%s::%s::%s::%s::%s", + hostUuid ?: "", + vmUuid ?: "", + purpose ?: "", + keyVersion == null ? "" : String.valueOf(keyVersion), + usage) + } +} diff --git a/testlib/src/main/java/org/zstack/testlib/Test.groovy b/testlib/src/main/java/org/zstack/testlib/Test.groovy index ba84a10c09a..696694fc6bb 100755 --- a/testlib/src/main/java/org/zstack/testlib/Test.groovy +++ b/testlib/src/main/java/org/zstack/testlib/Test.groovy @@ -28,6 +28,7 @@ import org.zstack.testlib.util.Retry import org.zstack.testlib.util.TestConfigUtils import org.zstack.utils.ShellUtils import org.zstack.utils.Utils +import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil import org.zstack.utils.logging.CLogger import org.zstack.utils.path.PathUtil @@ -1093,4 +1094,8 @@ mysqldump -u root zstack > ${failureLogDir.absolutePath}/dbdump.sql StartMode getCaseMode() { return StartMode.DEFAULT } + + static long gb(double gb) { + return SizeUnit.GIGABYTE.toByte(gb) + } } diff --git a/utils/src/main/java/org/zstack/utils/CollectionUtils.java b/utils/src/main/java/org/zstack/utils/CollectionUtils.java index b04a0635668..65e4f0e439d 100755 --- a/utils/src/main/java/org/zstack/utils/CollectionUtils.java +++ b/utils/src/main/java/org/zstack/utils/CollectionUtils.java @@ -5,6 +5,7 @@ import org.zstack.utils.function.ListFunction; import org.zstack.utils.logging.CLogger; +import javax.annotation.Nullable; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; @@ -64,6 +65,7 @@ public static List filter(Collection from, Predicate tester) { return from.stream().filter(tester).collect(Collectors.toList()); } + @Nullable public static T findOneOrNull(Collection from, Predicate tester) { return from.stream().filter(tester).findFirst().orElse(null); } diff --git a/utils/src/main/java/org/zstack/utils/StringDSL.java b/utils/src/main/java/org/zstack/utils/StringDSL.java index 6cd30a8122f..705b88a8b4d 100755 --- a/utils/src/main/java/org/zstack/utils/StringDSL.java +++ b/utils/src/main/java/org/zstack/utils/StringDSL.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -148,6 +149,21 @@ public static boolean isApiId(String apiId) { return apiId != null && apiId.matches("[0-9a-f]{8}[0-9a-f]{4}[0-9a-f]{4}[0-9a-f]{4}[0-9a-f]{12}"); } + public static String createFixedUuid(Class voClass) { + return createFixedUuid(voClass.getSimpleName()); + } + + public static String createFixedUuid(String text) { + StringBuilder builder = new StringBuilder( + UUID.nameUUIDFromBytes(text.getBytes()).toString().replace("-", "")); + builder.setCharAt(0, 'f'); + builder.setCharAt(1, 'f'); + builder.setCharAt(2, '0'); + builder.setCharAt(30, '0'); + builder.setCharAt(31, '0'); + return builder.toString(); + } + private static int hashOXR(List lst) { return lst.stream().mapToInt(Object::hashCode) .reduce(0, (l, r) -> l ^ r); diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 9fc175de2c1..403f1013b0a 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -1400,6 +1400,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_STORAGE_SNAPSHOT_10029 = "ORG_ZSTACK_STORAGE_SNAPSHOT_10029"; + public static final String ORG_ZSTACK_STORAGE_SNAPSHOT_10030 = "ORG_ZSTACK_STORAGE_SNAPSHOT_10030"; + public static final String ORG_ZSTACK_ZWATCH_NAMESPACE_10000 = "ORG_ZSTACK_ZWATCH_NAMESPACE_10000"; public static final String ORG_ZSTACK_ZWATCH_NAMESPACE_10001 = "ORG_ZSTACK_ZWATCH_NAMESPACE_10001"; @@ -2748,6 +2750,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_KVM_10162 = "ORG_ZSTACK_KVM_10162"; + public static final String ORG_ZSTACK_KVM_10163 = "ORG_ZSTACK_KVM_10163"; + public static final String ORG_ZSTACK_ZSV_10000 = "ORG_ZSTACK_ZSV_10000"; public static final String ORG_ZSTACK_ZSV_10001 = "ORG_ZSTACK_ZSV_10001"; @@ -11537,6 +11541,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_STORAGE_VOLUME_10100 = "ORG_ZSTACK_STORAGE_VOLUME_10100"; + public static final String ORG_ZSTACK_STORAGE_VOLUME_10101 = "ORG_ZSTACK_STORAGE_VOLUME_10101"; + public static final String ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_STORAGE_PRIMARY_BLOCK_VENDOR_XSTOR_10000 = "ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_STORAGE_PRIMARY_BLOCK_VENDOR_XSTOR_10000"; public static final String ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_STORAGE_PRIMARY_BLOCK_VENDOR_XSTOR_10001 = "ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_STORAGE_PRIMARY_BLOCK_VENDOR_XSTOR_10001"; @@ -12875,6 +12881,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_COMPUTE_VM_DEVICES_10008 = "ORG_ZSTACK_COMPUTE_VM_DEVICES_10008"; + public static final String ORG_ZSTACK_COMPUTE_VM_DEVICES_10009 = "ORG_ZSTACK_COMPUTE_VM_DEVICES_10009"; + public static final String ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_FAULTTOLERANCE_10000 = "ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_FAULTTOLERANCE_10000"; public static final String ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_FAULTTOLERANCE_10001 = "ORG_ZSTACK_TEST_INTEGRATION_PREMIUM_FAULTTOLERANCE_10001";