[ZSV-12866] migrate vTPM/KMS zstack support

compute/src/main/java/org/zstack/compute/legacy/ComputeLegacyGlobalProperty.java

@@ -0,0 +1,10 @@
+package org.zstack.compute.legacy;
+
+import org.zstack.core.GlobalProperty;
+import org.zstack.core.GlobalPropertyDefinition;
+
+@GlobalPropertyDefinition
+public class ComputeLegacyGlobalProperty {
+    @GlobalProperty(name="enable.nv.ram.type.volume", defaultValue = "false")
+    public static boolean enableNvRamTypeVolume;
+}

compute/src/main/java/org/zstack/compute/vm/InstantiateVmFromNewCreatedStruct.java

@@ -5,6 +5,7 @@
 import org.zstack.header.vm.InstantiateNewCreatedVmInstanceMsg;
 import org.zstack.header.vm.VmCreationStrategy;
 import org.zstack.header.vm.VmNicSpec;
+import org.zstack.header.vm.devices.VmDevicesSpec;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -31,6 +32,7 @@ public class InstantiateVmFromNewCreatedStruct {
     private List<String> sshKeyPairUuids;
     private final List<String> candidatePrimaryStorageUuidsForRootVolume = new ArrayList<>();
     private final List<String> candidatePrimaryStorageUuidsForDataVolume = new ArrayList<>();
+    private VmDevicesSpec devicesSpec;
 
     public List<String> getCandidatePrimaryStorageUuidsForRootVolume() {
         return candidatePrimaryStorageUuidsForRootVolume;
@@ -64,6 +66,14 @@ public void setDiskAOs(List<APICreateVmInstanceMsg.DiskAO> diskAOs) {
         this.diskAOs = diskAOs;
     }
 
+    public VmDevicesSpec getDevicesSpec() {
+        return devicesSpec;
+    }
+
+    public void setDevicesSpec(VmDevicesSpec devicesSpec) {
+        this.devicesSpec = devicesSpec;
+    }
+
     public List<String> getRootVolumeSystemTags() {
         return rootVolumeSystemTags;
     }
@@ -142,6 +152,7 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(InstantiateNewCreate
         struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
         struct.setDisableL3Networks(msg.getDisableL3Networks());
         struct.setDiskAOs(msg.getDiskAOs());
+        struct.setDevicesSpec(msg.getDevicesSpec());
         return struct;
     }
 
@@ -161,6 +172,7 @@ public static InstantiateVmFromNewCreatedStruct fromMessage(CreateVmInstanceMsg
         struct.setDataVolumeSystemTagsOnIndex(msg.getDataVolumeSystemTagsOnIndex());
         struct.setDisableL3Networks(msg.getDisableL3Networks());
         struct.setDiskAOs(msg.getDiskAOs());
+        struct.setDevicesSpec(msg.getDevicesSpec());
         return struct;
     }
 

compute/src/main/java/org/zstack/compute/vm/VmCascadeExtension.java

@@ -463,9 +463,11 @@ public void done(ErrorCodeList errorCodeList) {
                                 .flatMap(List::stream).map(VmCdRomInventory::getUuid)
                                 .collect(Collectors.toList());
                         dbf.removeByPrimaryKeys(cdRomUuids, VmCdRomVO.class);
-                        dbf.removeByPrimaryKeys(vminvs.stream().map(p -> p.getInventory().getUuid())
-                                        .collect(Collectors.toList()),
-                                VmInstanceVO.class);
+
+                        List<String> vmUuidList = vminvs.stream()
+                                .map(vm -> vm.getInventory().getUuid())
+                                .collect(Collectors.toList());
+                        dbf.removeByPrimaryKeys(vmUuidList, VmInstanceVO.class);
                     }
 
                     completion.success();

compute/src/main/java/org/zstack/compute/vm/VmDeleteVolumeFlow.java

@@ -47,7 +47,9 @@ public void run(final FlowTrigger trigger, Map data) {
         final boolean deleteDataDisk = VmGlobalConfig.DELETE_DATA_VOLUME_ON_VM_DESTROY.value(Boolean.class);
 
         /* data volume must be detached anyway no matter if it is going to be deleted */
-        if (spec.getVmInventory().getAllVolumes().size() > 1) {
+        boolean anyDataVolume = spec.getVmInventory().getAllVolumes().stream()
+                .anyMatch(arg -> VolumeType.Data.toString().equals(arg.getType()));
+        if (anyDataVolume) {
             detachDataVolumes(spec);
         }
 
@@ -58,7 +60,11 @@ public void run(final FlowTrigger trigger, Map data) {
             return;
         }
 
-        List<String> volumeTypes = Arrays.asList(VolumeType.Root.toString(), VolumeType.Memory.toString(), VolumeType.Cache.toString());
+        List<String> volumeTypes = Arrays.asList(
+                VolumeType.Root.toString(),
+                VolumeType.Memory.toString(),
+                VolumeType.Cache.toString()
+        );
         List<VolumeDeletionStruct> ctx = CollectionUtils.transformToList(spec.getVmInventory().getAllVolumes(), new Function<VolumeDeletionStruct, VolumeInventory>() {
             @Override
             public VolumeDeletionStruct call(VolumeInventory arg) {

compute/src/main/java/org/zstack/compute/vm/VmExpungeVmResourceCascadeFlow.java

@@ -0,0 +1,83 @@
+package org.zstack.compute.vm;
+
+import org.springframework.beans.factory.annotation.Autowire;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Configurable;
+import org.zstack.compute.vm.devices.VmTpmManager;
+import org.zstack.core.cascade.CascadeConstant;
+import org.zstack.core.cascade.CascadeFacade;
+import org.zstack.core.componentloader.PluginRegistry;
+import org.zstack.header.core.Completion;
+import org.zstack.header.core.workflow.FlowTrigger;
+import org.zstack.header.core.workflow.NoRollbackFlow;
+import org.zstack.header.errorcode.ErrorCode;
+import org.zstack.header.vm.VmAfterExpungeExtensionPoint;
+import org.zstack.header.vm.VmDeletionStruct;
+import org.zstack.header.vm.VmInstanceConstant;
+import org.zstack.header.vm.VmInstanceDeletionPolicyManager.VmInstanceDeletionPolicy;
+import org.zstack.header.vm.VmInstanceInventory;
+import org.zstack.header.vm.VmInstanceSpec;
+import org.zstack.header.vm.VmInstanceVO;
+import org.zstack.utils.CollectionUtils;
+import org.zstack.utils.Utils;
+import org.zstack.utils.logging.CLogger;
+
+import java.util.Map;
+
+import static org.zstack.utils.CollectionDSL.list;
+
+/**
+ * Runs after volume/metadata expunge flows: fires {@link VmAfterExpungeExtensionPoint}, then
+ * {@link CascadeConstant#VM_INSTANCE_EXPUNGE_CODE} so TPM / VM host-file rows are removed before the VM row is deleted.
+ */
+@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE)
+public class VmExpungeVmResourceCascadeFlow extends NoRollbackFlow {
+    private static final CLogger logger = Utils.getLogger(VmExpungeVmResourceCascadeFlow.class);
+
+    /**
+     * Snapshot of TPM uuid before cascade deletes {@link org.zstack.header.tpm.entity.TpmVO};
+     * read by {@link VmInstanceBase} expunge done handler for {@code detachTpmKeyProviderBestEffort}.
+     */
+    public static final String EXPUNGE_CASCADE_TPM_UUID_KEY =
+            VmExpungeVmResourceCascadeFlow.class.getSimpleName() + ".tpmUuidBeforeCascade";
+
+    @Autowired
+    private CascadeFacade casf;
+    @Autowired
+    private PluginRegistry pluginRgty;
+
+    @Override
+    public void run(FlowTrigger trigger, Map data) {
+        final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString());
+        if (spec == null || spec.getVmInventory() == null) {
+            logger.warn("[VmExpungeVmResourceCascadeFlow] missing VmInstanceSpec, skip cascade");
+            trigger.next();
+            return;
+        }
+
+        final VmInstanceInventory inv = spec.getVmInventory();
+
+        CollectionUtils.safeForEach(pluginRgty.getExtensionList(VmAfterExpungeExtensionPoint.class),
+                arg -> arg.vmAfterExpunge(inv));
+
+        final String tpmUuidBeforeCascade = VmTpmManager.findTpmUuidForVmOrNull(inv.getUuid());
+        data.put(EXPUNGE_CASCADE_TPM_UUID_KEY, tpmUuidBeforeCascade);
+
+        VmDeletionStruct expungeStruct = new VmDeletionStruct();
+        expungeStruct.setInventory(inv);
+        expungeStruct.setDeletionPolicy(VmInstanceDeletionPolicy.Direct);
+
+        casf.asyncCascade(CascadeConstant.VM_INSTANCE_EXPUNGE_CODE, VmInstanceVO.class.getSimpleName(),
+                list(expungeStruct), new Completion(trigger) {
+                    @Override
+                    public void success() {
+                        trigger.next();
+                    }
+
+                    @Override
+                    public void fail(ErrorCode errorCode) {
+                        trigger.fail(errorCode);
+                    }
+                });
+    }
+}

compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java

@@ -117,6 +117,7 @@ public class VmGlobalConfig {
     @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "sync clock after vm resume")
     public static GlobalConfig VM_CLOCK_SYNC_AFTER_VM_RESUME = new GlobalConfig(CATEGORY, "vm.clock.sync.after.vm.resume");
 
+    @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class})
     @GlobalConfigValidation(validValues = {"true", "false"})
     public static GlobalConfig ENABLE_UEFI_SECURE_BOOT = new GlobalConfig(CATEGORY, "enable.uefi.secure.boot");
 
@@ -141,4 +142,13 @@ public class VmGlobalConfig {
     @GlobalConfigValidation(validValues = {"true", "false", "auto"})
     @GlobalConfigDef(defaultValue = "false", type = String.class, description = "generate config required for vhost primary storage")
     public static GlobalConfig GENERATE_CONFIG_VHOST_REQUIRED = new GlobalConfig(CATEGORY, "generate.config.vhost.required");
+
+    @GlobalConfigDef(defaultValue = "true", type = Boolean.class, description = "whether reset TPM state after VM clone")
+    @GlobalConfigValidation(validValues = {"true", "false"})
+    @BindResourceConfig(value = {VmInstanceVO.class, ClusterVO.class})
+    public static GlobalConfig RESET_TPM_AFTER_VM_CLONE = new GlobalConfig(CATEGORY, "reset.tpm.after.vm.clone");
+
+    @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "allowed TPM VM start without KMS")
+    @GlobalConfigValidation(validValues = {"true", "false"})
+    public static GlobalConfig ALLOWED_TPM_VM_WITHOUT_KMS = new GlobalConfig(CATEGORY, "allowed.tpm.vm.without.kms");
 }

compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java

@@ -1094,7 +1094,8 @@ private void validateRootDiskOffering(ImageMediaType imgFormat, APICreateVmInsta
                 }
 
                 if (msg.getRootDiskSize() <= 0) {
-                    throw new ApiMessageInterceptionException(operr(ORG_ZSTACK_COMPUTE_VM_10182, "Unexpected root disk settings"));
+                    throw new ApiMessageInterceptionException(operr(ORG_ZSTACK_COMPUTE_VM_10182,
+                            "Unexpected root disk settings: DiskAO[0].size is mandatory when image format is ISO"));
                 }
             }
         }

compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java

@@ -7,6 +7,8 @@
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.transaction.annotation.Transactional;
 import org.zstack.compute.allocator.HostAllocatorManager;
+import org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend;
+import org.zstack.compute.vm.devices.VmTpmManager;
 import org.zstack.core.Platform;
 import org.zstack.core.asyncbatch.While;
 import org.zstack.core.cascade.CascadeConstant;
@@ -145,11 +147,26 @@ public class VmInstanceBase extends AbstractVmInstance {
     private TagManager tagMgr;
     @Autowired
     private VmInstanceDeviceManager vidm;
+    @Autowired
+    private TpmEncryptedResourceKeyBackend tpmKeyBackend;
 
     protected VmInstanceVO self;
     protected VmInstanceVO originalCopy;
     protected String syncThreadName;
 
+    private void detachTpmKeyProviderBestEffort(String tpmUuid) {
+        if (tpmUuid == null) {
+            return;
+        }
+        try {
+            tpmKeyBackend.detachKeyProviderFromTpm(tpmUuid);
+        } catch (Throwable t) {
+            logger.warn(String.format(
+                    "failed to detach key provider from TPM[uuid:%s]: %s",
+                    tpmUuid, t.getMessage()), t);
+        }
+    }
+
     protected void checkState(final String hostUuid, final NoErrorCompletion completion) {
         CheckVmStateOnHypervisorMsg msg = new CheckVmStateOnHypervisorMsg();
         msg.setVmInstanceUuids(list(self.getUuid()));
@@ -1439,21 +1456,9 @@ private void expunge(Message msg, final Completion completion) {
         chain.done(new FlowDoneHandler(completion) {
             @Override
             public void handle(Map data) {
-                CollectionUtils.safeForEach(pluginRgty.getExtensionList(VmAfterExpungeExtensionPoint.class),
-                        arg -> arg.vmAfterExpunge(inv));
-
-                callVmJustBeforeDeleteFromDbExtensionPoint();
-
-                dbf.reload(self);
-                dbf.removeCollection(self.getVmNics(), VmNicVO.class);
-                dbf.removeCollection(self.getVmCdRoms(), VmCdRomVO.class);
-                dbf.remove(self);
-                logger.debug(String.format("successfully expunged the vm[uuid:%s]", self.getUuid()));
-                dbf.eoCleanup(VmInstanceVO.class, self.getUuid());
-                if (inv.getRootVolumeUuid() != null) {
-                    dbf.eoCleanup(VolumeVO.class, inv.getRootVolumeUuid());
-                }
-                completion.success();
+                final String tpmUuidForEncryptedKeyRef =
+                        (String) data.get(VmExpungeVmResourceCascadeFlow.EXPUNGE_CASCADE_TPM_UUID_KEY);
+                finishExpungeAfterVmResourceCascade(inv, tpmUuidForEncryptedKeyRef, completion);
             }
         }).error(new FlowErrorHandler(completion) {
             @Override
@@ -1463,6 +1468,23 @@ public void handle(final ErrorCode errCode, Map data) {
         }).start();
     }
 
+    private void finishExpungeAfterVmResourceCascade(VmInstanceInventory inv, String tpmUuidForEncryptedKeyRef,
+            Completion completion) {
+        callVmJustBeforeDeleteFromDbExtensionPoint();
+
+        self = dbf.reload(self);
+        dbf.removeCollection(self.getVmNics(), VmNicVO.class);
+        dbf.removeCollection(self.getVmCdRoms(), VmCdRomVO.class);
+        dbf.remove(self);
+        logger.debug(String.format("successfully expunged the vm[uuid:%s]", self.getUuid()));
+        dbf.eoCleanup(VmInstanceVO.class, self.getUuid());
+        if (inv.getRootVolumeUuid() != null) {
+            dbf.eoCleanup(VolumeVO.class, inv.getRootVolumeUuid());
+        }
+        detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef);
+        completion.success();
+    }
+
     private void handle(final VmCheckOwnStateMsg msg) {
         thdf.chainSubmit(new ChainTask(msg) {
             @Override
@@ -2803,12 +2825,15 @@ public void success() {
                     if (self.getState() != VmInstanceState.Destroyed) {
                         changeVmStateInDb(VmInstanceStateEvent.destroyed);
                     }
+                    final String tpmUuidForEncryptedKeyRef = VmTpmManager.findTpmUuidForVmOrNull(self.getUuid());
                     callVmJustBeforeDeleteFromDbExtensionPoint();
                     dbf.removeCollection(self.getVmCdRoms(), VmCdRomVO.class);
                     dbf.remove(getSelf());
                     dbf.eoCleanup(VmInstanceVO.class, self.getUuid());
+                    detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef);
                 } else if (deletionPolicy == VmInstanceDeletionPolicy.DBOnly || deletionPolicy == VmInstanceDeletionPolicy.KeepVolume) {
                     String accountUuid = acntMgr.getOwnerAccountUuidOfResource(inv.getUuid());
+                    final String tpmUuidForEncryptedKeyRef = VmTpmManager.findTpmUuidForVmOrNull(self.getUuid());
                     new SQLBatch() {
                         @Override
                         protected void scripts() {
@@ -2822,6 +2847,7 @@ protected void scripts() {
                             sql(VmInstanceVO.class).eq(VmInstanceVO_.uuid, self.getUuid()).hardDelete();
                         }
                     }.execute();
+                    detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef);
                     callVmJustAfterDeleteFromDbExtensionPoint(inv, accountUuid);
                 } else if (deletionPolicy == VmInstanceDeletionPolicy.Delay) {
                     changeVmStateInDb(VmInstanceStateEvent.destroyed);
@@ -7767,6 +7793,7 @@ public DiskOfferingVO call(DiskOfferingVO arg) {
         }
 
         spec.setDiskAOs(struct.getDiskAOs());
+        spec.setDevicesSpec(struct.getDevicesSpec());
 
         List<CdRomSpec> cdRomSpecs = buildVmCdRomSpecsForNewCreated(spec);
         spec.setCdRomSpecs(cdRomSpecs);
@@ -7952,6 +7979,7 @@ public void handle(final Map data) {
         }).error(new FlowErrorHandler(completion) {
             @Override
             public void handle(final ErrorCode errCode, Map data) {
+                final String tpmUuidForEncryptedKeyRef = VmTpmManager.findTpmUuidForVmOrNull(self.getUuid());
                 extEmitter.failedToStartNewCreatedVm(VmInstanceInventory.valueOf(self), errCode);
                 dbf.remove(self);
                 // clean up EO, otherwise API-retry may cause conflict if
@@ -7962,6 +7990,7 @@ public void handle(final ErrorCode errCode, Map data) {
                     logger.warn(e.getMessage());
                 }
 
+                detachTpmKeyProviderBestEffort(tpmUuidForEncryptedKeyRef);
                 completion.fail(operr(ORG_ZSTACK_COMPUTE_VM_10289, errCode, errCode.getDetails()));
             }
         }).start();