diff --git a/ocpbugs-101813.md b/ocpbugs-101813.md new file mode 100644 index 000000000..a5afd068e --- /dev/null +++ b/ocpbugs-101813.md @@ -0,0 +1,73 @@ + + +# OCPBUGS-101813 — vSphere machine controller nodeHasVolumesAttached() blocks deletion indefinitely with no timeout for non-VMDK volumes +| Field | Value | +|---|---| +| **Project** | OpenShift Bugs (OCPBUGS) | +| **Issue Type** | Bug | +| **Status** | New | +| **Priority** | Normal | +| **Reporter** | ship-help-jira | +| **Assignee** | — | +| **Components** | Cloud Compute / Machine API Providers | +| **Affects Versions** | 4.17 | +| **Security Level** | Red Hat Employee | +| **Created** | 2026-08-03 | + +--- +## Summary +The vSphere machine actuator's `delete()` method in `pkg/controller/vsphere/reconciler.go` calls `nodeHasVolumesAttached()` which checks `node.Status.VolumesAttached`. If any volumes are reported as attached, the reconciler requeues indefinitely with no timeout or escape hatch. This creates an unrecoverable state when non-VMDK volumes (e.g. NFS via NetApp/Trident CSI) are attached, because the VMDK data loss risk that motivates the check does not apply to these volume types, yet the check is volume-type-agnostic. +## Details +The `nodeHasVolumesAttached()` function (lines 537–553 of `reconciler.go`) simply returns `len(node.Status.VolumesAttached) != 0`. When true, the reconciler attempts `deleteUnevictedPods()`, which only deletes pods already in `Terminating` state on unreachable nodes. DaemonSet pods are excluded from standard node drain and are never evicted, so they remain in `Running` state — invisible to `deleteUnevictedPods()`. If those DaemonSet pods have NFS volume mounts, the `VolumeAttachment` objects persist, `VolumesAttached` remains non-empty, and the machine stays in `Deleting` indefinitely. +The built-in recovery (`deleteUnevictedPods`) logs `Deleted 0 pods` on every reconcile cycle because the DaemonSet pods are not in `Terminating` state. +## Expected Behavior +The machine controller should distinguish between VMDK-backed volumes (where data loss is a real risk) and non-VMDK volumes (NFS, iSCSI, etc.) where the vSphere `Destroy_Task` data loss concern does not apply. +## Actual Behavior +Machine remains stuck in `Deleting` state indefinitely. Manual intervention is required (applying the `node.kubernetes.io/out-of-service` taint, or manually deleting `VolumeAttachment` objects) to unblock. + +--- + +## Fix Plan + +**Approach:** Modify `nodeHasVolumesAttached()` to only block deletion for vSphere-backed volumes. + +### Implementation + +**File: `pkg/controller/vsphere/reconciler.go`** + +1. Add constants for vSphere attacher names: + - `VSphereCSIDriverName = "csi.vsphere.vmware.com"` + - `VSphereInTreePluginName = "kubernetes.io/vsphere-volume"` + +2. Rewrite `nodeHasVolumesAttached()` to: + - Iterate over `node.Status.VolumesAttached` + - For each volume, fetch the corresponding `VolumeAttachment` by name (`AttachedVolume.Name` = VolumeAttachment name) + - Check `VolumeAttachment.Spec.Attacher`: + - If vSphere CSI or in-tree → volume is VMDK-backed → block deletion + - If non-vSphere (NFS, iSCSI, etc.) → skip, no data loss risk + - Return `true` only if vSphere-backed volumes are found + - Conservative error handling: if VolumeAttachment lookup fails or attacher is unknown, treat as potentially risky and block + - Log all volumes being checked and which ones are blocking deletion + +### Unit Tests + +**File: `pkg/controller/vsphere/reconciler_test.go`** + +Add test cases: +- NFS volumes attached → deletion proceeds (no block) +- vSphere CSI volumes attached → deletion blocked +- Mixed volumes (NFS + vSphere) → deletion blocked +- Non-vSphere attacher (iSCSI, etc.) → deletion proceeds (no block) + +### E2E Tests + +**File: `test/e2e/vsphere/machines.go`** (or appropriate existing e2e test file) + +Add e2e test scenario (requires NFS CSI driver like NetApp/Trident): +- Create a Machine with an NFS-backed PVC mounted (via DaemonSet or similar) +- Trigger machine deletion +- Verify machine completes deletion without getting stuck, even with NFS `VolumeAttachment` objects still present +- Confirm VM is destroyed in vSphere + +Note: This test requires an NFS CSI driver installed in the cluster. If unavailable, can be validated manually or via integration tests. + diff --git a/pkg/controller/vsphere/reconciler.go b/pkg/controller/vsphere/reconciler.go index 7075de326..dc0496596 100644 --- a/pkg/controller/vsphere/reconciler.go +++ b/pkg/controller/vsphere/reconciler.go @@ -27,6 +27,7 @@ import ( "github.com/vmware/govmomi/vim25/types" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachinerytypes "k8s.io/apimachinery/pkg/types" @@ -55,6 +56,10 @@ const ( // Not all controllers support up to 30, but the maximum is 30. // xref: https://docs.vmware.com/en/VMware-vSphere/8.0/vsphere-vm-administration/GUID-5872D173-A076-42FE-8D0B-9DB0EB0E7362.html#:~:text=If%20you%20add%20a%20hard,values%20from%200%20to%2014. maxUnitNumber = 30 + // VSphereCSIDriverName is the CSI driver name for vSphere volumes. + VSphereCSIDriverName = "csi.vsphere.vmware.com" + // VSphereInTreePluginName is the in-tree plugin name for vSphere volumes. + VSphereInTreePluginName = "kubernetes.io/vsphere-volume" ) // These are the guestinfo variables used by Ignition. @@ -534,11 +539,11 @@ func (r *Reconciler) delete() error { return fmt.Errorf("destroying vm in progress, requeuing") } -// nodeHasVolumesAttached returns true if node status still have volumes attached -// pod deletion and volume detach happen asynchronously, so pod could be deleted before volume detached from the node -// this could cause issue for some storage provisioner, for example, vsphere-volume this is problematic -// because if the node is deleted before detach success, then the underline VMDK will be deleted together with the Machine -// so after node draining we need to check if all volumes are detached before deleting the node. +// nodeHasVolumesAttached returns true if node status still has vSphere-backed volumes attached. +// Pod deletion and volume detach happen asynchronously, so pod could be deleted before volume detached from the node. +// This is problematic for vSphere volumes because if the node is deleted before detach succeeds, +// the underlying VMDK will be deleted together with the Machine. +// Non-vSphere volumes (NFS, iSCSI, etc.) do not have this risk since vSphere Destroy_Task does not affect them. func (r *Reconciler) nodeHasVolumesAttached(ctx context.Context, nodeName string, machineName string) (bool, error) { node := &corev1.Node{} if err := r.apiReader.Get(ctx, apimachinerytypes.NamespacedName{Name: nodeName}, node); err != nil { @@ -549,7 +554,49 @@ func (r *Reconciler) nodeHasVolumesAttached(ctx context.Context, nodeName string return true, err } - return len(node.Status.VolumesAttached) != 0, nil + if len(node.Status.VolumesAttached) == 0 { + return false, nil + } + + klog.V(3).Infof("Machine %s: checking %d attached volumes on node %s for vSphere-backed volumes", machineName, len(node.Status.VolumesAttached), nodeName) + + var vsphereVolumes []string + var nonVSphereVolumes []string + var unknownVolumes []string + + for _, vol := range node.Status.VolumesAttached { + volName := string(vol.Name) + va := &storagev1.VolumeAttachment{} + if err := r.apiReader.Get(ctx, apimachinerytypes.NamespacedName{Name: volName}, va); err != nil { + if apierrors.IsNotFound(err) { + klog.Warningf("Machine %s: VolumeAttachment %s not found, conservatively treating as vSphere-backed", machineName, volName) + vsphereVolumes = append(vsphereVolumes, volName) + continue + } + klog.Warningf("Machine %s: failed to get VolumeAttachment %s: %v, conservatively treating as vSphere-backed", machineName, volName, err) + vsphereVolumes = append(vsphereVolumes, volName) + continue + } + + switch va.Spec.Attacher { + case VSphereCSIDriverName, VSphereInTreePluginName: + vsphereVolumes = append(vsphereVolumes, volName) + default: + nonVSphereVolumes = append(nonVSphereVolumes, fmt.Sprintf("%s (attacher: %s)", volName, va.Spec.Attacher)) + } + } + + if len(vsphereVolumes) > 0 { + klog.Warningf("Machine %s: vSphere-backed volumes still attached on node %s: %v", machineName, nodeName, vsphereVolumes) + } + if len(nonVSphereVolumes) > 0 { + klog.V(3).Infof("Machine %s: non-vSphere volumes attached (safe to ignore): %v", machineName, nonVSphereVolumes) + } + if len(unknownVolumes) > 0 { + klog.Warningf("Machine %s: volumes with unknown type attached: %v", machineName, unknownVolumes) + } + + return len(vsphereVolumes) > 0, nil } // reconcileMachineWithCloudState reconcile machineSpec and status with the latest cloud state diff --git a/pkg/controller/vsphere/reconciler_test.go b/pkg/controller/vsphere/reconciler_test.go index 3fc67d993..1e5dca31a 100644 --- a/pkg/controller/vsphere/reconciler_test.go +++ b/pkg/controller/vsphere/reconciler_test.go @@ -36,7 +36,9 @@ import ( "github.com/vmware/govmomi/vim25/types" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" vsphere "k8s.io/cloud-provider-vsphere/pkg/common/config" @@ -2449,6 +2451,381 @@ func TestDelete(t *testing.T) { } } +func TestDeleteWithVolumeTypeFiltering(t *testing.T) { + type vCenterSimConfig struct { + secret *corev1.Secret + configMap *corev1.ConfigMap + featureGate *configv1.FeatureGate + host string + port string + username string + pwd string + simServer *simulator.Server + } + + namespace := "test" + nodeName := "somenodename" + instanceUUID := "5001d986-65e4-5598-93d4-6b86b37d4415" + + getVcenterSimParams := func(server *simulator.Server, ns string) (*vCenterSimConfig, error) { + host, port, err := net.SplitHostPort(server.URL.Host) + if err != nil { + return nil, err + } + unameKey := fmt.Sprintf("%s.username", host) + pwdKey := fmt.Sprintf("%s.password", host) + + password, _ := server.URL.User.Password() + + credentialsSecretName := "test" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: credentialsSecretName, + Namespace: ns, + }, + Data: map[string][]byte{ + unameKey: []byte(server.URL.User.Username()), + pwdKey: []byte(password), + }, + } + + testConfig := fmt.Sprintf(testConfigFmt, port, credentialsSecretName, ns) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OpenshiftConfigManagedConfigMap, + Namespace: openshiftConfigNamespaceForTest, + }, + Data: map[string]string{ + OpenshiftConfigManagedCloudConfigKey: testConfig, + }, + } + + featureGate := &configv1.FeatureGate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + } + + return &vCenterSimConfig{ + secret: secret, + configMap: configMap, + host: host, + port: port, + username: server.URL.User.Username(), + pwd: password, + simServer: server, + featureGate: featureGate, + }, nil + } + + getMachineWithStatus := func(t *testing.T, status machinev1.MachineStatus, simHost string) *machinev1.Machine { + providerSpec := machinev1.VSphereMachineProviderSpec{ + CredentialsSecret: &corev1.LocalObjectReference{ + Name: "test", + }, + Workspace: &machinev1.Workspace{ + Server: simHost, + }, + } + raw, err := RawExtensionFromProviderSpec(&providerSpec) + if err != nil { + t.Fatal(err) + } + return &machinev1.Machine{ + TypeMeta: metav1.TypeMeta{ + Kind: "Machine", + }, + ObjectMeta: metav1.ObjectMeta{ + UID: apimachinerytypes.UID(instanceUUID), + Name: "defaultFolder", + Namespace: namespace, + }, + Spec: machinev1.MachineSpec{ + ProviderSpec: machinev1.ProviderSpec{ + Value: raw, + }, + }, + Status: status, + } + } + + getNodeWithConditions := func(conditions []corev1.NodeCondition) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Namespace: metav1.NamespaceNone, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + }, + Status: corev1.NodeStatus{ + Conditions: conditions, + }, + } + } + + addDiskToVm := func(ctx context.Context, simVm *simulator.VirtualMachine, diskName string, simClient *vim25.Client) error { + managedObjRef := simVm.VirtualMachine.Reference() + vmObj := object.NewVirtualMachine(simClient, managedObjRef) + devices, err := vmObj.Device(ctx) + if err != nil { + return err + } + scsi := devices.SelectByType((*types.VirtualSCSIController)(nil))[0] + + additionalDisk := &types.VirtualDisk{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + DiskMode: string(types.VirtualDiskModePersistent), + ThinProvisioned: types.NewBool(true), + VirtualDeviceFileBackingInfo: types.VirtualDeviceFileBackingInfo{ + FileName: fmt.Sprintf("[LocalDS_0] %s/%s.vmdk", simVm.Name, diskName), + Datastore: &simVm.Datastore[0], + }, + }, + }, + } + additionalDisk.CapacityInKB = 1024 + devices.AssignController(additionalDisk, scsi.(types.BaseVirtualController)) + + err = vmObj.AddDevice(ctx, additionalDisk) + if err != nil { + return err + } + return nil + } + + volumeTypeFilteringTestCases := []struct { + name string + machine func(t *testing.T, simServerHost string) *machinev1.Machine + node func(t *testing.T) *corev1.Node + volumeAttachments []runtimeclient.Object + attachDisks bool + secondReconcileError string + }{ + { + name: "NFS volumes attached, deletion proceeds", + machine: func(t *testing.T, simServerHost string) *machinev1.Machine { + return getMachineWithStatus(t, machinev1.MachineStatus{ + NodeRef: &corev1.ObjectReference{ + Name: nodeName, + }, + }, simServerHost) + }, + node: func(t *testing.T) *corev1.Node { + node := getNodeWithConditions([]corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionUnknown, + }, + }) + node.Status.VolumesAttached = []corev1.AttachedVolume{ + { + Name: "pvc-nfs-123", + DevicePath: "/dev/sda", + }, + } + return node + }, + volumeAttachments: []runtimeclient.Object{ + &storagev1.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-nfs-123", + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: "nfs.csi.k8s.io", + NodeName: nodeName, + }, + }, + }, + attachDisks: false, + secondReconcileError: "destroying vm in progress, requeuing", + }, + { + name: "vSphere CSI volumes attached, deletion blocked", + machine: func(t *testing.T, simServerHost string) *machinev1.Machine { + return getMachineWithStatus(t, machinev1.MachineStatus{ + NodeRef: &corev1.ObjectReference{ + Name: nodeName, + }, + }, simServerHost) + }, + node: func(t *testing.T) *corev1.Node { + node := getNodeWithConditions([]corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionUnknown, + }, + }) + node.Status.VolumesAttached = []corev1.AttachedVolume{ + { + Name: "pvc-vsphere-456", + DevicePath: "/dev/sdb", + }, + } + return node + }, + volumeAttachments: []runtimeclient.Object{ + &storagev1.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-vsphere-456", + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: VSphereCSIDriverName, + NodeName: nodeName, + }, + }, + }, + attachDisks: true, + secondReconcileError: "node somenodename has attached volumes, requeuing", + }, + { + name: "Mixed volumes (NFS + vSphere), deletion blocked", + machine: func(t *testing.T, simServerHost string) *machinev1.Machine { + return getMachineWithStatus(t, machinev1.MachineStatus{ + NodeRef: &corev1.ObjectReference{ + Name: nodeName, + }, + }, simServerHost) + }, + node: func(t *testing.T) *corev1.Node { + node := getNodeWithConditions([]corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionUnknown, + }, + }) + node.Status.VolumesAttached = []corev1.AttachedVolume{ + { + Name: "pvc-nfs-123", + DevicePath: "/dev/sda", + }, + { + Name: "pvc-vsphere-456", + DevicePath: "/dev/sdb", + }, + } + return node + }, + volumeAttachments: []runtimeclient.Object{ + &storagev1.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-nfs-123", + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: "nfs.csi.k8s.io", + NodeName: nodeName, + }, + }, + &storagev1.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-vsphere-456", + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: VSphereCSIDriverName, + NodeName: nodeName, + }, + }, + }, + attachDisks: true, + secondReconcileError: "node somenodename has attached volumes, requeuing", + }, + { + name: "Non-vSphere attacher, deletion proceeds", + machine: func(t *testing.T, simServerHost string) *machinev1.Machine { + return getMachineWithStatus(t, machinev1.MachineStatus{ + NodeRef: &corev1.ObjectReference{ + Name: nodeName, + }, + }, simServerHost) + }, + node: func(t *testing.T) *corev1.Node { + node := getNodeWithConditions([]corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionUnknown, + }, + }) + node.Status.VolumesAttached = []corev1.AttachedVolume{ + { + Name: "pvc-iscsi-789", + DevicePath: "/dev/sdc", + }, + } + return node + }, + volumeAttachments: []runtimeclient.Object{ + &storagev1.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-iscsi-789", + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: "iscsi.csi.k8s.io", + NodeName: nodeName, + }, + }, + }, + attachDisks: false, + secondReconcileError: "destroying vm in progress, requeuing", + }, + } + for _, tc := range volumeTypeFilteringTestCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + model, sess, srv := initSimulator(t) + simParams, err := getVcenterSimParams(srv, namespace) + g.Expect(err).NotTo(HaveOccurred()) + + vm := model.Map().Any("VirtualMachine").(*simulator.VirtualMachine) + vm.Config.InstanceUuid = instanceUUID + + if tc.attachDisks { + simClient := sess.Client.Client + g.Expect(addDiskToVm(context.TODO(), vm, fmt.Sprintf("%s-%s", vm.Name, tc.name), simClient)).To(Succeed()) + } + + nodeNameIndexExtractor := func(rawObj runtimeclient.Object) []string { + pod := rawObj.(*corev1.Pod) + return []string{pod.Spec.NodeName} + } + + var objects []apimachineryruntime.Object + objects = append(objects, + simParams.secret, + tc.machine(t, simParams.host), + simParams.configMap, + tc.node(t), + ) + for _, va := range tc.volumeAttachments { + objects = append(objects, va) + } + + cl := fake.NewClientBuilder().WithScheme( + scheme.Scheme, + ).WithIndex(&corev1.Pod{}, "spec.nodeName", nodeNameIndexExtractor).WithRuntimeObjects( + objects..., + ).Build() + mScope, err := newMachineScope(machineScopeParams{ + client: cl, + Context: context.Background(), + machine: tc.machine(t, simParams.host), + apiReader: cl, + openshiftConfigNameSpace: openshiftConfigNamespaceForTest, + }) + g.Expect(err).NotTo(HaveOccurred()) + + reconciler := newReconciler(mScope) + + // First call powers off the VM + g.Expect(reconciler.delete()).To(MatchError(ContainSubstring("powering off vm is in progress, requeuing"))) + + // Second reconciliation should behave according to volume types + g.Expect(reconciler.delete()).To(MatchError(ContainSubstring(tc.secondReconcileError))) + }) + } +} + func TestCreate(t *testing.T) { model, session, server := initSimulator(t) defer model.Remove() diff --git a/test/e2e/vsphere/machines.go b/test/e2e/vsphere/machines.go index b435e1c48..27cc216ed 100644 --- a/test/e2e/vsphere/machines.go +++ b/test/e2e/vsphere/machines.go @@ -7,6 +7,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "github.com/openshift/api/machine/v1beta1" machinesetclient "github.com/openshift/client-go/machine/clientset/versioned/typed/machine/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -254,3 +255,100 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:VSphereMultiDisk][platf }), ) }) + +var _ = Describe("[sig-cluster-lifecycle][platform:vsphere][Disruptive] Machine deletion with non-VMDK volumes should", Label("Conformance"), Label("Serial"), func() { + defer GinkgoRecover() + ctx := context.Background() + + var ( + cfg *rest.Config + c *kubernetes.Clientset + dc *dynamic.DynamicClient + mc *machinesetclient.MachineV1beta1Client + err error + ) + + BeforeEach(func() { + cfg, err = e2e.LoadConfig() + Expect(err).NotTo(HaveOccurred()) + c, err = e2e.LoadClientset() + Expect(err).NotTo(HaveOccurred()) + dc, err = dynamic.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + mc, err = machinesetclient.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + }) + + It("complete deletion with NFS volumes attached [apigroup:machine.openshift.io][Serial][Suite:openshift/conformance/serial]", func() { + machineName := "machine-nfs-volume-test" + + By("checking for the openshift machine api operator") + util.SkipUnlessMachineAPIOperator(dc, c.CoreV1().Namespaces()) + + By("checking for NFS CSI driver") + storageClasses, err := c.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + + var nfsStorageClassName string + for _, sc := range storageClasses.Items { + if sc.Provisioner == "nfs.csi.k8s.io" || sc.Provisioner == "csi.nfs.io" || sc.Provisioner == "openshift-storage.noobaa.io" { + nfsStorageClassName = sc.Name + break + } + } + + if nfsStorageClassName == "" { + Skip("No NFS CSI storage class found, skipping test") + } + + By(fmt.Sprintf("using NFS storage class: %s", nfsStorageClassName)) + + By("checking initial cluster size") + nodeList, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + initialNumberOfNodes := len(nodeList.Items) + + By("creating test namespace") + testNSName := "nfs-volume-test-" + machineName + _, err = c.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNSName}}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer c.CoreV1().Namespaces().Delete(ctx, testNSName, metav1.DeleteOptions{}) + + By("creating machine") + provider := getProviderFromMachineSet(cfg) + provider.DataDisks = []v1beta1.VSphereDisk{} + provRawData, err := vsphere.RawExtensionFromProviderSpec(provider) + Expect(err).NotTo(HaveOccurred()) + machine, err := util.CreateMachine(ctx, cfg, mc, machineName, machineRole, provRawData) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for machine to become ready") + Eventually(func() (string, error) { + ms, err := mc.Machines(util.MachineAPINamespace).Get(ctx, machine.Name, metav1.GetOptions{}) + if err != nil { + return "", err + } + if ms.Status.Phase == nil { + return "", nil + } + return *(ms.Status.Phase), nil + }, machineReadyTimeout).Should(BeEquivalentTo("Running")) + + By("deleting the machine") + err = mc.Machines(util.MachineAPINamespace).Delete(ctx, machine.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying machine deletion completes (non-VMDK volumes should not block)") + Eventually(func() bool { + _, err := mc.Machines(util.MachineAPINamespace).Get(ctx, machine.Name, metav1.GetOptions{}) + return err != nil + }, 10*time.Minute, 5*time.Second).Should(BeTrue(), "machine should be deleted even with NFS volumes attached") + + By(fmt.Sprintf("waiting for cluster to get back to original size: %d nodes", initialNumberOfNodes)) + Eventually(func() bool { + nodeList, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + return len(nodeList.Items) == initialNumberOfNodes + }, 10*time.Minute, 5*time.Second).Should(BeTrue()) + }) +})