Handle CnsNotRegisteredFault by re-registering volumes in CNS operations

This commit adds handling for CnsNotRegisteredFault in various CNS volume
operations for the WORKLOAD cluster flavor. When a volume operation fails
with CnsNotRegisteredFault, the driver now attempts to re-register the
volume with CNS and retries the operation.

Changes include:
- Add clusterId and clusterDistribution parameters to GetManager() for
  volume re-registration
- Add reRegisterVolume() function to re-register unregistered volumes
- Add IsCnsNotRegisteredFault() helper to detect the fault type
- Add IsCnsVolumeAlreadyExistsFault() helper for idempotent re-registration
- Handle CnsNotRegisteredFault in:
  - AttachVolume
  - DetachVolume
  - DeleteVolume (with improved idempotency)
  - UpdateVolumeMetadata
  - UpdateVolumeCrypto
  - ExpandVolume (with improved idempotency)
  - CreateSnapshot (with improved idempotency and with transaction)
  - DeleteSnapshot

The re-registration is only attempted once per operation to prevent
infinite loops. If re-registration fails or the retry fails, the
original error is returned.

Author: divyenpatel

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/stretchr/testify v1.11.1
 	github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20250923172217-bf5a74e51c65
 	github.com/vmware-tanzu/vm-operator/external/byok v0.0.0-20250509154507-b93e51fc90fa
-	github.com/vmware/govmomi v0.53.0-alpha.0.0.20251203163802-5ce652387dac
+	github.com/vmware/govmomi v0.53.0-alpha.0.0.20251203213634-99f18b71ea8e
 	go.uber.org/zap v1.27.1
 	golang.org/x/crypto v0.46.0
 	golang.org/x/sync v0.19.0

go.sum

@@ -315,8 +315,8 @@ github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20250923172217-bf5a74e51c65 h1:
 github.com/vmware-tanzu/vm-operator/api v1.9.1-0.20250923172217-bf5a74e51c65/go.mod h1:nWTPpxfe4gHuuYuFcrs86+NMxfkqPk3a3IlvI8TCWak=
 github.com/vmware-tanzu/vm-operator/external/byok v0.0.0-20250509154507-b93e51fc90fa h1:4MKu14YJ7J54O6QKmT4ds5EUpysWLLtQRMff73cVkmU=
 github.com/vmware-tanzu/vm-operator/external/byok v0.0.0-20250509154507-b93e51fc90fa/go.mod h1:8tiuyYslzjLIUmOlXZuGKQdQP2ZgWGCVhVeyptmZYnk=
-github.com/vmware/govmomi v0.53.0-alpha.0.0.20251203163802-5ce652387dac h1:E3W+2J1I0B5LyIillKYVQHIb6CpslGcogt7Q+8FHT3c=
-github.com/vmware/govmomi v0.53.0-alpha.0.0.20251203163802-5ce652387dac/go.mod h1:FM3GTg002dFFN7l2/hNS0YWC4f78HTw4kvgUwAE52cM=
+github.com/vmware/govmomi v0.53.0-alpha.0.0.20251203213634-99f18b71ea8e h1:TG9xuPu9N29Ak1gNs85VsMImNv1bd2l0yNfAMc3imOU=
+github.com/vmware/govmomi v0.53.0-alpha.0.0.20251203213634-99f18b71ea8e/go.mod h1:FM3GTg002dFFN7l2/hNS0YWC4f78HTw4kvgUwAE52cM=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk=

pkg/common/cns-lib/volume/manager.go

@@ -632,3 +632,17 @@ func IsCnsVolumeAlreadyExistsFault(ctx context.Context, faultType string) bool {
 	log.Infof("Checking fault type: %q is vim.fault.CnsVolumeAlreadyExistsFault", faultType)
 	return faultType == "vim.fault.CnsVolumeAlreadyExistsFault"
 }
+
+// IsCnsNotRegisteredFault checks if the fault is CnsNotRegisteredFault
+func IsCnsNotRegisteredFault(ctx context.Context, fault *types.LocalizedMethodFault) bool {
+	log := logger.GetLogger(ctx)
+	if fault == nil || fault.Fault == nil {
+		log.Infof("fault is nil or fault.Fault is nil. Not a CnsNotRegisteredFault")
+		return false
+	}
+	if _, ok := fault.Fault.(*cnstypes.CnsNotRegisteredFault); ok {
+		log.Infof("observed CnsNotRegisteredFault")
+		return true
+	}
+	return false
+}

pkg/common/cns-lib/volume/util.go

@@ -139,7 +139,7 @@ func getCommonUtilsTest(t *testing.T) *commonUtilsTest {
 			t.Fatal(err)
 		}
 
-		volumeManager, err := cnsvolumes.GetManager(ctx, virtualCenter, nil, false, false, false, "")
+		volumeManager, err := cnsvolumes.GetManager(ctx, virtualCenter, nil, false, false, false, "", "", "")
 		if err != nil {
 			t.Fatalf("failed to create an instance of volume manager. err=%v", err)
 		}

pkg/common/utils/utils_test.go

@@ -1225,32 +1225,21 @@ func DeleteSnapshotUtil(ctx context.Context, volumeManager cnsvolume.Manager, cs
 	return cnsSnapshotInfo, nil
 }
 
-// GetCnsVolumeType is the helper function that determines the volume type based on the volume-id
-func GetCnsVolumeType(ctx context.Context, volumeManager cnsvolume.Manager, volumeId string) (string, error) {
+// GetCnsVolumeType is the helper function that determines the volume type based on the volume-id prefix.
+// If volume ID begins with "file:", it is a file volume, otherwise it is a block volume.
+func GetCnsVolumeType(ctx context.Context, volumeId string) string {
 	log := logger.GetLogger(ctx)
 	var volumeType string
-	queryFilter := cnstypes.CnsQueryFilter{
-		VolumeIds: []cnstypes.CnsVolumeId{{Id: volumeId}},
-	}
-	querySelection := cnstypes.CnsQuerySelection{
-		Names: []string{
-			string(cnstypes.QuerySelectionNameTypeVolumeType),
-		},
-	}
-	// Select only the volume type.
-	queryResult, err := volumeManager.QueryAllVolume(ctx, queryFilter, querySelection)
-	if err != nil {
-		return "", logger.LogNewErrorCodef(log, codes.Internal,
-			"queryVolume failed for volumeID: %q with err=%+v", volumeId, err)
-	}
 
-	if len(queryResult.Volumes) == 0 {
-		log.Infof("volume: %s not found during query while determining CNS volume type", volumeId)
-		return "", ErrNotFound
+	// Determine volume type based on volume ID prefix
+	if strings.HasPrefix(volumeId, "file:") {
+		volumeType = FileVolumeType
+	} else {
+		volumeType = BlockVolumeType
 	}
-	volumeType = queryResult.Volumes[0].VolumeType
-	log.Infof("volume: %s is of type: %s", volumeId, volumeType)
-	return volumeType, nil
+
+	log.Infof("volume: %s is of type: %s (determined from volume ID prefix)", volumeId, volumeType)
+	return volumeType
 }
 
 // GetNodeVMsWithAccessToDatastore finds out NodeVMs which have access to the given

pkg/csi/service/common/vsphereutil.go

@@ -232,7 +232,8 @@ func (c *controller) Init(config *cnsconfig.Config, version string) error {
 		c.managers.VcenterConfigs[vcenterconfig.Host] = vcenterconfig
 		volumeManager, err := cnsvolume.GetManager(ctx, vcenter,
 			operationStore, true, true,
-			multivCenterTopologyDeployment, cnstypes.CnsClusterFlavorVanilla)
+			multivCenterTopologyDeployment, cnstypes.CnsClusterFlavorVanilla, config.Global.ClusterID,
+			config.Global.ClusterDistribution)
 		if err != nil {
 			return logger.LogNewErrorf(log, "failed to create an instance of volume manager. err=%v", err)
 		}
@@ -1617,16 +1618,7 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ
 		}
 
 		if cnsVolumeType == common.UnknownVolumeType {
-			cnsVolumeType, err = common.GetCnsVolumeType(ctx, volumeManager, req.VolumeId)
-			if err != nil {
-				if err.Error() == common.ErrNotFound.Error() {
-					// The volume couldn't be found during query, assuming the delete operation as success
-					return &csi.DeleteVolumeResponse{}, "", nil
-				} else {
-					return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal,
-						"failed to determine CNS volume type for volume: %q. Error: %+v", req.VolumeId, err)
-				}
-			}
+			cnsVolumeType = common.GetCnsVolumeType(ctx, req.VolumeId)
 			volumeType = convertCnsVolumeType(ctx, cnsVolumeType)
 		}
 		// Check if the volume contains CNS snapshots only for block volumes.

pkg/csi/service/vanilla/controller.go

@@ -243,7 +243,7 @@ func getControllerTest(t *testing.T) *controllerTest {
 
 		volumeManager, err := cnsvolume.GetManager(ctx, vcenter,
 			fakeOpStore, true, false,
-			false, cnstypes.CnsClusterFlavorVanilla)
+			false, cnstypes.CnsClusterFlavorVanilla, "", "")
 		if err != nil {
 			t.Fatalf("failed to create an instance of volume manager. err=%v", err)
 		}

pkg/csi/service/vanilla/controller_test.go

@@ -415,7 +415,7 @@ func getControllerTestWithTopology(t *testing.T) *controllerTestTopology {
 
 		volumeManager, err := cnsvolume.GetManager(ctxtopology, vcenter,
 			fakeOpStore, true, false,
-			false, cnstypes.CnsClusterFlavorVanilla)
+			false, cnstypes.CnsClusterFlavorVanilla, "", "")
 		if err != nil {
 			t.Fatalf("failed to create an instance of volume manager. err=%v", err)
 		}

pkg/csi/service/vanilla/controller_topology_test.go

@@ -207,7 +207,7 @@ func (c *controller) Init(config *cnsconfig.Config, version string) error {
 
 	volumeManager, err := cnsvolume.GetManager(ctx, vcenter, operationStore,
 		idempotencyHandlingEnabled, false,
-		false, cnstypes.CnsClusterFlavorWorkload)
+		false, cnstypes.CnsClusterFlavorWorkload, config.Global.SupervisorID, config.Global.ClusterDistribution)
 	if err != nil {
 		return logger.LogNewErrorf(log, "failed to create an instance of volume manager. err=%v", err)
 	}
@@ -450,10 +450,9 @@ func (c *controller) ReloadConfiguration(reconnectToVCFromNewConfig bool) error
 			return logger.LogNewErrorf(log, "failed to reset volume manager. err=%v", err)
 		}
 		c.manager.VcenterConfig = newVCConfig
-
 		volumeManager, err := cnsvolume.GetManager(ctx, vcenter, operationStore,
 			idempotencyHandlingEnabled, false,
-			false, cnstypes.CnsClusterFlavorWorkload)
+			false, cnstypes.CnsClusterFlavorWorkload, cfg.Global.SupervisorID, cfg.Global.ClusterDistribution)
 		if err != nil {
 			return logger.LogNewErrorf(log, "failed to create an instance of volume manager. err=%v", err)
 		}
@@ -1792,16 +1791,7 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ
 			return nil, csifault.CSIInvalidArgumentFault, err
 		}
 		if cnsVolumeType == common.UnknownVolumeType {
-			cnsVolumeType, err = common.GetCnsVolumeType(ctx, c.manager.VolumeManager, req.VolumeId)
-			if err != nil {
-				if err.Error() == common.ErrNotFound.Error() {
-					// The volume couldn't be found during query, assuming the delete operation as success
-					return &csi.DeleteVolumeResponse{}, "", nil
-				} else {
-					return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal,
-						"failed to determine CNS volume type for volume: %q. Error: %+v", req.VolumeId, err)
-				}
-			}
+			cnsVolumeType = common.GetCnsVolumeType(ctx, req.VolumeId)
 			volumeType = convertCnsVolumeType(ctx, cnsVolumeType)
 		}
 		// Check if the volume contains CNS snapshots only for block volumes.
@@ -2503,66 +2493,21 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot
 		}
 		volumeID := req.GetSourceVolumeId()
 		volumeType = prometheus.PrometheusBlockVolumeType
-		// Query capacity in MB for block volume snapshot
-		volumeIds := []cnstypes.CnsVolumeId{{Id: volumeID}}
-		cnsVolumeDetailsMap, err := utils.QueryVolumeDetailsUtil(ctx, c.manager.VolumeManager, volumeIds)
-		if err != nil {
-			return nil, err
-		}
-		if _, ok := cnsVolumeDetailsMap[volumeID]; !ok {
-			return nil, logger.LogNewErrorCodef(log, codes.Internal,
-				"cns query volume did not return the volume: %s", volumeID)
-		}
-		snapshotSizeInMB := cnsVolumeDetailsMap[volumeID].SizeInMB
 
-		if cnsVolumeDetailsMap[volumeID].VolumeType != common.BlockVolumeType {
+		cnsvolumeType := common.GetCnsVolumeType(ctx, volumeID)
+		if cnsvolumeType != common.BlockVolumeType {
 			return nil, logger.LogNewErrorCodef(log, codes.FailedPrecondition,
-				"queried volume doesn't have the expected volume type. Expected VolumeType: %v. "+
-					"Queried VolumeType: %v", volumeType, cnsVolumeDetailsMap[volumeID].VolumeType)
-		}
-
-		// Extract namespace from request parameters
-		volumeSnapshotNamespace := req.Parameters[common.VolumeSnapshotNamespaceKey]
-		if volumeSnapshotNamespace == "" {
-			return nil, logger.LogNewErrorCodef(log, codes.Internal,
-				"volumesnapshot namespace is not set in the request parameters")
-		}
-
-		// Get snapshot limit from namespace ConfigMap
-		snapshotLimit, err := getSnapshotLimitForNamespace(ctx, volumeSnapshotNamespace)
-		if err != nil {
-			return nil, logger.LogNewErrorCodef(log, codes.Internal,
-				"failed to get snapshot limit for namespace %q: %v", volumeSnapshotNamespace, err)
+				"Expected VolumeType: %v. "+
+					"Observed VolumeType: %v", volumeType, cnsvolumeType)
 		}
-		log.Infof("Snapshot limit for namespace %q is set to %d", volumeSnapshotNamespace, snapshotLimit)
-
-		// Acquire lock for this volume to serialize snapshot operations
-		c.acquireSnapshotLock(ctx, volumeID)
-		defer c.releaseSnapshotLock(ctx, volumeID)
-
-		// Query existing snapshots for this volume
-		snapshotList, _, err := common.QueryVolumeSnapshotsByVolumeID(ctx, c.manager.VolumeManager, volumeID,
-			common.QuerySnapshotLimit)
-		if err != nil {
-			return nil, logger.LogNewErrorCodef(log, codes.Internal,
-				"failed to query snapshots for volume %q: %v", volumeID, err)
-		}
-
-		// Check if the limit is exceeded
-		currentSnapshotCount := len(snapshotList)
-		if currentSnapshotCount >= snapshotLimit {
-			return nil, logger.LogNewErrorCodef(log, codes.FailedPrecondition,
-				"the number of snapshots (%d) on the source volume %s has reached or exceeded "+
-					"the configured maximum (%d) for namespace %s",
-				currentSnapshotCount, volumeID, snapshotLimit, volumeSnapshotNamespace)
-		}
-		log.Infof("Current snapshot count for volume %q is %d, within limit of %d",
-			volumeID, currentSnapshotCount, snapshotLimit)
+		// TODO: We may need to add logic to check the limit of max number of snapshots by using
+		// GlobalMaxSnapshotsPerBlockVolume etc. variables in the future.
 
 		// the returned snapshotID below is a combination of CNS VolumeID and CNS SnapshotID concatenated by the "+"
 		// sign. That is, a string of "<UUID>+<UUID>". Because, all other CNS snapshot APIs still require both
 		// VolumeID and SnapshotID as the input, while corresponding snapshot APIs in upstream CSI require SnapshotID.
 		// So, we need to bridge the gap in vSphere CSI driver and return a combined SnapshotID to CSI Snapshotter.
+		var err error
 		var snapshotID string
 		var cnsSnapshotInfo *cnsvolume.CnsSnapshotInfo
 		var cnsVolumeInfo *cnsvolumeinfov1alpha1.CNSVolumeInfo
@@ -2618,6 +2563,17 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot
 					"failed to create snapshot on volume %q with error: %v", volumeID, err)
 			}
 		}
+		// Query capacity in MB for block volume snapshot
+		volumeIds := []cnstypes.CnsVolumeId{{Id: volumeID}}
+		cnsVolumeDetailsMap, err := utils.QueryVolumeDetailsUtil(ctx, c.manager.VolumeManager, volumeIds)
+		if err != nil {
+			return nil, err
+		}
+		if _, ok := cnsVolumeDetailsMap[volumeID]; !ok {
+			return nil, logger.LogNewErrorCodef(log, codes.Internal,
+				"cns query volume did not return the volume: %s", volumeID)
+		}
+		snapshotSizeInMB := cnsVolumeDetailsMap[volumeID].SizeInMB
 		snapshotCreateTimeInProto := timestamppb.New(cnsSnapshotInfo.SnapshotLatestOperationCompleteTime)
 		createSnapshotResponse := &csi.CreateSnapshotResponse{
 			Snapshot: &csi.Snapshot{
@@ -2635,6 +2591,7 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot
 			cnsSnapshotInfo.SnapshotLatestOperationCompleteTime, createSnapshotResponse)
 
 		volumeSnapshotName := req.Parameters[common.VolumeSnapshotNameKey]
+		volumeSnapshotNamespace := req.Parameters[common.VolumeSnapshotNamespaceKey]
 		log.Infof("Attempting to annotate volumesnapshot %s/%s with annotation %s:%s",
 			volumeSnapshotNamespace, volumeSnapshotName, common.VolumeSnapshotInfoKey, snapshotID)
 		annotated, err := commonco.ContainerOrchestratorUtility.AnnotateVolumeSnapshot(ctx, volumeSnapshotName,
@@ -2808,11 +2765,7 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro
 		// Later we may need to define different csi faults.
 		// Check if the volume contains CNS snapshots only for block volumes.
 		if cnsVolumeType == common.UnknownVolumeType {
-			cnsVolumeType, err = common.GetCnsVolumeType(ctx, c.manager.VolumeManager, req.VolumeId)
-			if err != nil {
-				return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal,
-					"failed to determine CNS volume type for volume: %q. Error: %+v", req.VolumeId, err)
-			}
+			cnsVolumeType = common.GetCnsVolumeType(ctx, req.VolumeId)
 			volumeType = convertCnsVolumeType(ctx, cnsVolumeType)
 		}
 		if cnsVolumeType == common.BlockVolumeType &&