From 71b0721e6eb2ec425bfb0a5e7da4029c50f8d8d3 Mon Sep 17 00:00:00 2001 From: Niclas Schad Date: Tue, 4 Aug 2026 12:20:54 +0200 Subject: [PATCH 1/3] WIP: Delete Volume when creation fails with status ERROR Signed-off-by: Niclas Schad --- pkg/csi/blockstorage/controllerserver.go | 17 +++++++++++++++-- pkg/stackit/client/iaas.go | 21 ++++++++++++++------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pkg/csi/blockstorage/controllerserver.go b/pkg/csi/blockstorage/controllerserver.go index 367d8340..e975a6c5 100644 --- a/pkg/csi/blockstorage/controllerserver.go +++ b/pkg/csi/blockstorage/controllerserver.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "strconv" + "strings" "time" "github.com/container-storage-interface/spec/lib/go/csi" @@ -265,14 +266,26 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol targetStatus := []string{stackitclient.VolumeAvailableStatus} // Recheck after: 0s (immediate), 20s, 45.6s, 78.36s, 120.31s - err = cloud.WaitVolumeTargetStatusWithCustomBackoff(ctx, *vol.Id, targetStatus, + err = cloud.WaitVolumeTargetStatusWithCustomBackoff(ctx, &vol, targetStatus, &wait.Backoff{ Duration: 20 * time.Second, Steps: 5, Factor: 1.28, }) if err != nil { - klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", *vol.Id, err) + klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", vol.GetId(), err) + + if err != nil { + klog.Errorf("Failed to fetch volume %s status during cleanup check: %v", vol.GetId(), err) + } else if strings.ToUpper(vol.GetStatus()) == stackitclient.VolumeErrorStatus { + klog.Warningf("Volume %s entered ERROR status, attempting cleanup deletion...", vol.GetId()) + if deleteErr := cloud.DeleteVolume(ctx, vol.GetId()); deleteErr != nil { + klog.Errorf("Failed to delete erroneous volume %s: %v", vol.GetId(), deleteErr) + } else { + klog.Infof("Successfully deleted erroneous volume %s", vol.GetId()) + } + } + return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume Volume %s failed getting available in time: %v", *vol.Id, err)) } diff --git a/pkg/stackit/client/iaas.go b/pkg/stackit/client/iaas.go index 56ed1588..0da8fc84 100644 --- a/pkg/stackit/client/iaas.go +++ b/pkg/stackit/client/iaas.go @@ -49,12 +49,13 @@ type IaaSClient interface { WaitVolumeTargetStatus(ctx context.Context, volumeID string, tStatus []string) error WaitDiskAttached(ctx context.Context, instanceID, volumeID string) error WaitDiskDetached(ctx context.Context, instanceID, volumeID string) error - WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, volumeID string, tStatus []string, backoff *wait.Backoff) error + WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, vol **iaas.Volume, tStatus []string, backoff *wait.Backoff) error } const ( VolumeAvailableStatus = "AVAILABLE" VolumeAttachedStatus = "ATTACHED" + VolumeErrorStatus = "ERROR" operationFinishInitDelay = 1 * time.Second operationFinishFactor = 1.1 operationFinishSteps = 10 @@ -542,25 +543,31 @@ func (i *iaasClient) DetachVolume(ctx context.Context, serverID, volumeID string return nil } -func (i *iaasClient) WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, volumeID string, tStatus []string, backoff *wait.Backoff) error { +func (i *iaasClient) WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, vol **iaas.Volume, tStatus []string, backoff *wait.Backoff) error { + volID := (*vol).GetId() + waitErr := wait.ExponentialBackoff(*backoff, func() (bool, error) { - vol, err := i.GetVolume(ctx, volumeID) + updatedVol, err := i.GetVolume(ctx, volID) if err != nil { return false, err } - if slices.Contains(tStatus, *vol.Status) { + + // Update vol so we can skip having another request + *vol = updatedVol + + if slices.Contains(tStatus, updatedVol.GetStatus()) { return true, nil } for _, eState := range volumeErrorStates { - if *vol.Status == eState { - return false, fmt.Errorf("volume is in error state: %s", *vol.Status) + if updatedVol.GetStatus() == eState { + return false, fmt.Errorf("volume is in error state: %s", updatedVol.GetStatus()) } } return false, nil }) if wait.Interrupted(waitErr) { - waitErr = fmt.Errorf("timeout on waiting for volume %s status to be in %v", volumeID, tStatus) + waitErr = fmt.Errorf("timeout on waiting for volume %s status to be in %v", volID, tStatus) } return waitErr From e5c57faff601dd4d3f4ea040d592033f8fe45435 Mon Sep 17 00:00:00 2001 From: Niclas Schad Date: Tue, 4 Aug 2026 13:33:40 +0200 Subject: [PATCH 2/3] cleanup if logic Signed-off-by: Niclas Schad --- pkg/csi/blockstorage/controllerserver.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/csi/blockstorage/controllerserver.go b/pkg/csi/blockstorage/controllerserver.go index e975a6c5..47593f87 100644 --- a/pkg/csi/blockstorage/controllerserver.go +++ b/pkg/csi/blockstorage/controllerserver.go @@ -274,10 +274,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }) if err != nil { klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", vol.GetId(), err) - - if err != nil { - klog.Errorf("Failed to fetch volume %s status during cleanup check: %v", vol.GetId(), err) - } else if strings.ToUpper(vol.GetStatus()) == stackitclient.VolumeErrorStatus { + if strings.ToUpper(vol.GetStatus()) == stackitclient.VolumeErrorStatus { klog.Warningf("Volume %s entered ERROR status, attempting cleanup deletion...", vol.GetId()) if deleteErr := cloud.DeleteVolume(ctx, vol.GetId()); deleteErr != nil { klog.Errorf("Failed to delete erroneous volume %s: %v", vol.GetId(), deleteErr) @@ -285,7 +282,6 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol klog.Infof("Successfully deleted erroneous volume %s", vol.GetId()) } } - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume Volume %s failed getting available in time: %v", *vol.Id, err)) } From 447c883aa8ef40add3a4c22629247a2e783c2f50 Mon Sep 17 00:00:00 2001 From: Niclas Schad Date: Tue, 4 Aug 2026 13:50:39 +0200 Subject: [PATCH 3/3] move deleteVolumesInErrorState behind CLI flag Signed-off-by: Niclas Schad --- cmd/stackit-csi-plugin/main.go | 22 ++++++++++++-------- pkg/csi/blockstorage/controllerserver.go | 22 ++++++++++++-------- pkg/csi/blockstorage/driver.go | 26 +++++++++++++++--------- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/cmd/stackit-csi-plugin/main.go b/cmd/stackit-csi-plugin/main.go index 302aee97..534a5583 100644 --- a/cmd/stackit-csi-plugin/main.go +++ b/cmd/stackit-csi-plugin/main.go @@ -23,14 +23,15 @@ import ( ) var ( - endpoint string - cloudConfig string - cluster string - metricsAddress string - provideControllerService bool - provideNodeService bool - legacyStorageMode bool - legacyVolumeCreation bool + endpoint string + cloudConfig string + cluster string + metricsAddress string + provideControllerService bool + provideNodeService bool + legacyStorageMode bool + legacyVolumeCreation bool + deleteVolumesInErrorState bool ) func main() { @@ -85,6 +86,7 @@ func main() { cmd.PersistentFlags().BoolVar(&legacyStorageMode, "legacy-storage-mode", false, "Configures the CSI to listen to the legacy storage driverName cinder.csi.openstack.org instead") cmd.PersistentFlags().BoolVar(&legacyVolumeCreation, "legacy-volume-creation", true, "Enable or disable support for creating volumes with the old driverName (cinder.csi.openstack.org)") + cmd.PersistentFlags().BoolVar(&deleteVolumesInErrorState, "delete-volumes-in-error", false, "Delete volumes in error state when creating") stackitclient.AddExtraFlags(pflag.CommandLine) @@ -117,6 +119,10 @@ func handle(ctx context.Context) { driverOpts.BlockVolumeCreation = true } + if deleteVolumesInErrorState { + driverOpts.DeleteVolumesInErrorState = true + } + d := blockstorage.NewDriver(driverOpts) if provideControllerService { diff --git a/pkg/csi/blockstorage/controllerserver.go b/pkg/csi/blockstorage/controllerserver.go index 47593f87..fa668905 100644 --- a/pkg/csi/blockstorage/controllerserver.go +++ b/pkg/csi/blockstorage/controllerserver.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "strconv" - "strings" "time" "github.com/container-storage-interface/spec/lib/go/csi" @@ -274,13 +273,8 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }) if err != nil { klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", vol.GetId(), err) - if strings.ToUpper(vol.GetStatus()) == stackitclient.VolumeErrorStatus { - klog.Warningf("Volume %s entered ERROR status, attempting cleanup deletion...", vol.GetId()) - if deleteErr := cloud.DeleteVolume(ctx, vol.GetId()); deleteErr != nil { - klog.Errorf("Failed to delete erroneous volume %s: %v", vol.GetId(), deleteErr) - } else { - klog.Infof("Successfully deleted erroneous volume %s", vol.GetId()) - } + if cs.Driver.deleteVolumesInErrorState { + cs.deleteVolumeInError(ctx, vol) } return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume Volume %s failed getting available in time: %v", *vol.Id, err)) } @@ -290,6 +284,18 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return cs.getCreateVolumeResponse(vol), nil } +func (cs *controllerServer) deleteVolumeInError(ctx context.Context, vol *iaas.Volume) { + cloud := cs.Instance + if vol.GetStatus() == stackitclient.VolumeErrorStatus { + klog.Warningf("Volume %s entered ERROR status, attempting cleanup deletion...", vol.GetId()) + if deleteErr := cloud.DeleteVolume(ctx, vol.GetId()); deleteErr != nil { + klog.Errorf("Failed to delete erroneous volume %s: %v", vol.GetId(), deleteErr) + } else { + klog.Infof("Successfully deleted erroneous volume %s", vol.GetId()) + } + } +} + func setVolumeEncryptionParameters(opts *iaas.CreateVolumePayload, volParams *stackitParameterConfig) error { err := validateEncryptionConfig(volParams) if err != nil { diff --git a/pkg/csi/blockstorage/driver.go b/pkg/csi/blockstorage/driver.go index 60e3b6f9..41948708 100644 --- a/pkg/csi/blockstorage/driver.go +++ b/pkg/csi/blockstorage/driver.go @@ -31,12 +31,13 @@ var ( ) type Driver struct { - name string - fqVersion string // Fully qualified version in format {Version}@{CPO version} - endpoint string - clusterID string - legacyDriver bool - blockVolumeCreation bool + name string + fqVersion string // Fully qualified version in format {Version}@{CPO version} + endpoint string + clusterID string + legacyDriver bool + blockVolumeCreation bool + deleteVolumesInErrorState bool ids *identityServer cs *controllerServer @@ -51,10 +52,11 @@ type Driver struct { } type DriverOpts struct { - ClusterID string - Endpoint string - LegacyDriverName bool - BlockVolumeCreation bool + ClusterID string + Endpoint string + LegacyDriverName bool + BlockVolumeCreation bool + DeleteVolumesInErrorState bool PVCLister corev1.PersistentVolumeClaimLister } @@ -73,6 +75,10 @@ func NewDriver(o *DriverOpts) *Driver { d.legacyDriver = true } + if o.DeleteVolumesInErrorState { + d.deleteVolumesInErrorState = true + } + if o.BlockVolumeCreation { d.blockVolumeCreation = true }