WIP: Delete Volume when creation fails with status ERROR

cmd/stackit-csi-plugin/main.go

@@ -23,14 +23,15 @@ import (
 )
 
 var (
-	endpoint                 string
-	cloudConfig              string
-	cluster                  string
-	metricsAddress           string
-	provideControllerService bool
-	provideNodeService       bool
-	legacyStorageMode        bool
-	legacyVolumeCreation     bool
+	endpoint                  string
+	cloudConfig               string
+	cluster                   string
+	metricsAddress            string
+	provideControllerService  bool
+	provideNodeService        bool
+	legacyStorageMode         bool
+	legacyVolumeCreation      bool
+	deleteVolumesInErrorState bool
 )
 
 func main() {
@@ -85,6 +86,7 @@ func main() {
 	cmd.PersistentFlags().BoolVar(&legacyStorageMode, "legacy-storage-mode", false,
 		"Configures the CSI to listen to the legacy storage driverName cinder.csi.openstack.org instead")
 	cmd.PersistentFlags().BoolVar(&legacyVolumeCreation, "legacy-volume-creation", true, "Enable or disable support for creating volumes with the old driverName (cinder.csi.openstack.org)")
+	cmd.PersistentFlags().BoolVar(&deleteVolumesInErrorState, "delete-volumes-in-error", false, "Delete volumes in error state when creating")
 
 	stackitclient.AddExtraFlags(pflag.CommandLine)
 
@@ -117,6 +119,10 @@ func handle(ctx context.Context) {
 		driverOpts.BlockVolumeCreation = true
 	}
 
+	if deleteVolumesInErrorState {
+		driverOpts.DeleteVolumesInErrorState = true
+	}
+
 	d := blockstorage.NewDriver(driverOpts)
 
 	if provideControllerService {

pkg/csi/blockstorage/controllerserver.go

@@ -265,14 +265,17 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 
 	targetStatus := []string{stackitclient.VolumeAvailableStatus}
 	// Recheck after: 0s (immediate), 20s, 45.6s, 78.36s, 120.31s
-	err = cloud.WaitVolumeTargetStatusWithCustomBackoff(ctx, *vol.Id, targetStatus,
+	err = cloud.WaitVolumeTargetStatusWithCustomBackoff(ctx, &vol, targetStatus,
 		&wait.Backoff{
 			Duration: 20 * time.Second,
 			Steps:    5,
 			Factor:   1.28,
 		})
 	if err != nil {
-		klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", *vol.Id, err)
+		klog.Errorf("Failed to WaitVolumeTargetStatus of volume %s: %v", vol.GetId(), err)
+		if cs.Driver.deleteVolumesInErrorState {
+			cs.deleteVolumeInError(ctx, vol)
+		}
 		return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume Volume %s failed getting available in time: %v", *vol.Id, err))
 	}
 
@@ -281,6 +284,18 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	return cs.getCreateVolumeResponse(vol), nil
 }
 
+func (cs *controllerServer) deleteVolumeInError(ctx context.Context, vol *iaas.Volume) {
+	cloud := cs.Instance
+	if vol.GetStatus() == stackitclient.VolumeErrorStatus {
+		klog.Warningf("Volume %s entered ERROR status, attempting cleanup deletion...", vol.GetId())
+		if deleteErr := cloud.DeleteVolume(ctx, vol.GetId()); deleteErr != nil {
+			klog.Errorf("Failed to delete erroneous volume %s: %v", vol.GetId(), deleteErr)
+		} else {
+			klog.Infof("Successfully deleted erroneous volume %s", vol.GetId())
+		}
+	}
+}
+
 func setVolumeEncryptionParameters(opts *iaas.CreateVolumePayload, volParams *stackitParameterConfig) error {
 	err := validateEncryptionConfig(volParams)
 	if err != nil {

pkg/csi/blockstorage/driver.go

@@ -31,12 +31,13 @@ var (
 )
 
 type Driver struct {
-	name                string
-	fqVersion           string // Fully qualified version in format {Version}@{CPO version}
-	endpoint            string
-	clusterID           string
-	legacyDriver        bool
-	blockVolumeCreation bool
+	name                      string
+	fqVersion                 string // Fully qualified version in format {Version}@{CPO version}
+	endpoint                  string
+	clusterID                 string
+	legacyDriver              bool
+	blockVolumeCreation       bool
+	deleteVolumesInErrorState bool
 
 	ids *identityServer
 	cs  *controllerServer
@@ -51,10 +52,11 @@ type Driver struct {
 }
 
 type DriverOpts struct {
-	ClusterID           string
-	Endpoint            string
-	LegacyDriverName    bool
-	BlockVolumeCreation bool
+	ClusterID                 string
+	Endpoint                  string
+	LegacyDriverName          bool
+	BlockVolumeCreation       bool
+	DeleteVolumesInErrorState bool
 
 	PVCLister corev1.PersistentVolumeClaimLister
 }
@@ -73,6 +75,10 @@ func NewDriver(o *DriverOpts) *Driver {
 		d.legacyDriver = true
 	}
 
+	if o.DeleteVolumesInErrorState {
+		d.deleteVolumesInErrorState = true
+	}
+
 	if o.BlockVolumeCreation {
 		d.blockVolumeCreation = true
 	}

pkg/stackit/client/iaas.go

@@ -49,12 +49,13 @@ type IaaSClient interface {
 	WaitVolumeTargetStatus(ctx context.Context, volumeID string, tStatus []string) error
 	WaitDiskAttached(ctx context.Context, instanceID, volumeID string) error
 	WaitDiskDetached(ctx context.Context, instanceID, volumeID string) error
-	WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, volumeID string, tStatus []string, backoff *wait.Backoff) error
+	WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, vol **iaas.Volume, tStatus []string, backoff *wait.Backoff) error
 }
 
 const (
 	VolumeAvailableStatus    = "AVAILABLE"
 	VolumeAttachedStatus     = "ATTACHED"
+	VolumeErrorStatus        = "ERROR"
 	operationFinishInitDelay = 1 * time.Second
 	operationFinishFactor    = 1.1
 	operationFinishSteps     = 10
@@ -542,25 +543,31 @@ func (i *iaasClient) DetachVolume(ctx context.Context, serverID, volumeID string
 	return nil
 }
 
-func (i *iaasClient) WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, volumeID string, tStatus []string, backoff *wait.Backoff) error {
+func (i *iaasClient) WaitVolumeTargetStatusWithCustomBackoff(ctx context.Context, vol **iaas.Volume, tStatus []string, backoff *wait.Backoff) error {
+	volID := (*vol).GetId()
+
 	waitErr := wait.ExponentialBackoff(*backoff, func() (bool, error) {
-		vol, err := i.GetVolume(ctx, volumeID)
+		updatedVol, err := i.GetVolume(ctx, volID)
 		if err != nil {
 			return false, err
 		}
-		if slices.Contains(tStatus, *vol.Status) {
+
+		// Update vol so we can skip having another request
+		*vol = updatedVol
+
+		if slices.Contains(tStatus, updatedVol.GetStatus()) {
 			return true, nil
 		}
 		for _, eState := range volumeErrorStates {
-			if *vol.Status == eState {
-				return false, fmt.Errorf("volume is in error state: %s", *vol.Status)
+			if updatedVol.GetStatus() == eState {
+				return false, fmt.Errorf("volume is in error state: %s", updatedVol.GetStatus())
 			}
 		}
 		return false, nil
 	})
 
 	if wait.Interrupted(waitErr) {
-		waitErr = fmt.Errorf("timeout on waiting for volume %s status to be in %v", volumeID, tStatus)
+		waitErr = fmt.Errorf("timeout on waiting for volume %s status to be in %v", volID, tStatus)
 	}
 
 	return waitErr