Driver Upgrade Timeout (ROCm#304)

Author: sriram-30

api/v1alpha1/deviceconfig_types.go

@@ -455,6 +455,7 @@ type ModuleStatus struct {
 	KernelVersion      string       `json:"kernelVersion,omitempty"`
 	LastTransitionTime string       `json:"lastTransitionTime,omitempty"`
 	Status             UpgradeState `json:"status,omitempty"`
+	UpgradeStartTime   string       `json:"upgradeStartTime,omitempty"`
 }
 
 // DeviceConfigStatus defines the observed state of Module.

bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml

@@ -30,7 +30,7 @@ metadata:
         }
       ]
     capabilities: Basic Install
-    createdAt: "2024-12-14T11:49:07Z"
+    createdAt: "2024-12-18T02:20:28Z"
     operatorframework.io/suggested-namespace: kube-amd-gpu
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

bundle/manifests/amd.com_deviceconfigs.yaml

@@ -564,6 +564,8 @@ spec:
                       description: UpgradeState captures the state of the upgrade
                         process on a node
                       type: string
+                    upgradeStartTime:
+                      type: string
                   type: object
                 description: NodeModuleStatus contains per node status of driver module
                   installation

config/crd/bases/amd.com_deviceconfigs.yaml

@@ -560,6 +560,8 @@ spec:
                       description: UpgradeState captures the state of the upgrade
                         process on a node
                       type: string
+                    upgradeStartTime:
+                      type: string
                   type: object
                 description: NodeModuleStatus contains per node status of driver module
                   installation

helm-charts-k8s/Chart.lock

@@ -6,4 +6,4 @@ dependencies:
   repository: file://./charts/kmm
   version: v1.0.0
 digest: sha256:f9a315dd2ce3d515ebf28c8e9a6a82158b493ca2686439ec381487761261b597
-generated: "2024-12-14T12:29:21.567399824Z"
+generated: "2024-12-18T02:20:17.803725581Z"

helm-charts-k8s/crds/deviceconfig-crd.yaml

@@ -565,6 +565,8 @@ spec:
                       description: UpgradeState captures the state of the upgrade process
                         on a node
                       type: string
+                    upgradeStartTime:
+                      type: string
                   type: object
                 description: NodeModuleStatus contains per node status of driver module
                   installation

helm-charts-openshift/Chart.lock

@@ -6,4 +6,4 @@ dependencies:
   repository: file://./charts/kmm
   version: v1.0.0
 digest: sha256:25200c34a5cc846a1275e5bf3fc637b19e909dc68de938189c5278d77d03f5ac
-generated: "2024-12-14T11:49:06.370435251Z"
+generated: "2024-12-18T02:20:27.600366626Z"

helm-charts-openshift/crds/deviceconfig-crd.yaml

@@ -565,6 +565,8 @@ spec:
                       description: UpgradeState captures the state of the upgrade process
                         on a node
                       type: string
+                    upgradeStartTime:
+                      type: string
                   type: object
                 description: NodeModuleStatus contains per node status of driver module
                   installation

internal/controllers/device_config_reconciler.go

@@ -479,14 +479,27 @@ func (dcrh *deviceConfigReconcilerHelper) getDeviceConfigOwnedKMMModule(ctx cont
 
 func (dcrh *deviceConfigReconcilerHelper) updateDeviceConfigNodeStatus(ctx context.Context, devConfig *amdv1alpha1.DeviceConfig, nodes *v1.NodeList) error {
 	logger := log.FromContext(ctx)
+	previousUpgradeTimes := make(map[string]string)
+	// Persist the UpgradeStartTime
+	for nodeName, moduleStatus := range devConfig.Status.NodeModuleStatus {
+		previousUpgradeTimes[nodeName] = moduleStatus.UpgradeStartTime
+	}
 	devConfig.Status.NodeModuleStatus = map[string]amdv1alpha1.ModuleStatus{}
 
 	// for each node, fetch its status of modules configured by given DeviceConfig
 	for _, node := range nodes.Items {
 		// if there is no module configured for given node
-		// the info under that node name will have only status
+		// the info under that node name will have only status and upgrade start time
 		// then it will be clear to see which node didn't get module configured
-		devConfig.Status.NodeModuleStatus[node.Name] = amdv1alpha1.ModuleStatus{Status: dcrh.upgradeMgrHandler.GetNodeStatus(node.Name)}
+		upgradeStartTime := previousUpgradeTimes[node.Name]
+
+		currentStatus := dcrh.upgradeMgrHandler.GetNodeStatus(node.Name)
+		if currentStatus == amdv1alpha1.UpgradeStateFailed || currentStatus == amdv1alpha1.UpgradeStateCordonFailed || currentStatus == amdv1alpha1.UpgradeStateUncordonFailed || currentStatus == amdv1alpha1.UpgradeStateDrainFailed || currentStatus == amdv1alpha1.UpgradeStateRebootFailed || currentStatus == amdv1alpha1.UpgradeStateComplete || currentStatus == amdv1alpha1.UpgradeStateInstallComplete {
+			upgradeStartTime = ""
+		} else if upgradeStartTime == "" {
+			upgradeStartTime = dcrh.upgradeMgrHandler.GetNodeUpgradeStartTime(node.Name)
+		}
+		devConfig.Status.NodeModuleStatus[node.Name] = amdv1alpha1.ModuleStatus{Status: dcrh.upgradeMgrHandler.GetNodeStatus(node.Name), UpgradeStartTime: upgradeStartTime}
 
 		nmc := kmmv1beta1.NodeModulesConfig{}
 		err := dcrh.client.Get(ctx, types.NamespacedName{Name: node.Name}, &nmc)
@@ -507,6 +520,7 @@ func (dcrh *deviceConfigReconcilerHelper) updateDeviceConfigNodeStatus(ctx conte
 						KernelVersion:      module.Config.KernelVersion,
 						LastTransitionTime: module.LastTransitionTime.String(),
 						Status:             dcrh.upgradeMgrHandler.GetNodeStatus(node.Name),
+						UpgradeStartTime:   upgradeStartTime,
 					}
 				}
 			}