Merge pull request #18 from OpenSlides/feature/mschieder-bugfixes

Bugfixes for instance health progress bar

Author: 4echow

README.md

@@ -348,7 +348,6 @@ Applied namespace: myinstancedirorg
 Applying stack manifests from: my.instance.dir.org/stack/
 ...
 
-Waiting for instance to become ready:
 [████████████████████████████████████████] Pods ready (13/13)
 
 Instance is healthy: 13/13 pods ready

internal/constants/constants.go

@@ -108,6 +108,9 @@ const (
 	BarStart         string        = "["
 	BarEnd           string        = "]"
 	ThrottleDuration time.Duration = 100 * time.Millisecond
+	SpinnerType      int           = 14
+	// add extra line at the end of progress bar detail buffer for pending pod names
+	AddDetailLineBuffer int = 1
 
 	// wait function settings
 	TickerDuration time.Duration = 2 * time.Second // checks health conditions every tick

internal/k8s/actions/apply.go

@@ -38,6 +38,11 @@ func applyManifest(ctx context.Context, k8sClient *client.Client, manifestPath s
 		return "", fmt.Errorf("parsing YAML: %w", err)
 	}
 
+	if obj.GetKind() == "" {
+		logger.Info("Skipping manifest with no kind: %s", manifestPath)
+		return "", nil
+	}
+
 	namespace := obj.GetNamespace()
 	if namespace == "" && obj.GetKind() == "Namespace" {
 		namespace = obj.GetName()

internal/k8s/actions/health_check.go

@@ -3,6 +3,8 @@ package actions
 import (
 	"context"
 	"fmt"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/OpenSlides/openslides-cli/internal/constants"
@@ -31,7 +33,15 @@ func getHealthStatus(ctx context.Context, k8sClient *client.Client, namespace st
 		return nil, fmt.Errorf("listing pods: %w", err)
 	}
 
-	total := len(pods.Items)
+	var filteredPods []corev1.Pod
+	for _, pod := range pods.Items {
+		if pod.Status.Phase == corev1.PodSucceeded {
+			continue
+		}
+		filteredPods = append(filteredPods, pod)
+	}
+
+	total := len(filteredPods)
 	if total == 0 {
 		return &HealthStatus{
 			Healthy: false,
@@ -42,17 +52,19 @@ func getHealthStatus(ctx context.Context, k8sClient *client.Client, namespace st
 	}
 
 	ready := 0
-	for _, pod := range pods.Items {
+	for _, pod := range filteredPods {
 		if isPodReady(&pod) {
 			ready++
 		}
 	}
 
+	healthy := ready == total
+
 	return &HealthStatus{
-		Healthy: ready == total,
+		Healthy: healthy,
 		Ready:   ready,
 		Total:   total,
-		Pods:    pods.Items,
+		Pods:    filteredPods,
 	}, nil
 }
 
@@ -97,8 +109,6 @@ func checkHealth(ctx context.Context, k8sClient *client.Client, namespace string
 
 // waitForInstanceHealthy waits for instance to become healthy
 func waitForInstanceHealthy(ctx context.Context, k8sClient *client.Client, namespace string, timeout time.Duration) error {
-	logger.Info("Waiting for instance to become healthy (timeout: %v)", timeout)
-
 	ticker := time.NewTicker(constants.TickerDuration)
 	defer ticker.Stop()
 
@@ -119,17 +129,27 @@ func waitForInstanceHealthy(ctx context.Context, k8sClient *client.Client, names
 			lastStatus = status
 
 			if bar == nil && status.Total > 0 {
-				bar = createProgressBar(status.Total, "Pods ready")
+				bar = createProgressBar(status.Total, "Pods ready", constants.AddDetailLineBuffer)
+			} else if bar != nil {
+				bar.ChangeMax(status.Total)
 			}
-
-			if bar != nil {
+			if bar != nil && !bar.IsFinished() {
+				notReady := getNotReadyNames(status.Pods)
+				if len(notReady) > 0 {
+					if err := bar.AddDetail(fmt.Sprintf("%s Pending: %s", constants.IconNotReady, strings.Join(notReady, ", "))); err != nil {
+						return fmt.Errorf("adding pending pods detail: %w", err)
+					}
+				} else {
+					if err := bar.AddDetail(""); err != nil {
+						return fmt.Errorf("adding empty detail: %w", err)
+					}
+				}
 				if err := bar.Set(status.Ready); err != nil {
 					return fmt.Errorf("setting progress bar: %w", err)
 				}
 			}
-
 			if status.Healthy {
-				if bar != nil {
+				if bar != nil && !bar.IsFinished() {
 					if err := bar.Finish(); err != nil {
 						return fmt.Errorf("finishing progress bar: %w", err)
 					}
@@ -139,7 +159,7 @@ func waitForInstanceHealthy(ctx context.Context, k8sClient *client.Client, names
 			}
 
 		case <-timeoutCtx.Done():
-			if bar != nil {
+			if bar != nil && !bar.IsFinished() {
 				if err := bar.Finish(); err != nil {
 					return fmt.Errorf("finishing progress bar: %w", err)
 				}
@@ -153,32 +173,64 @@ func waitForInstanceHealthy(ctx context.Context, k8sClient *client.Client, names
 	}
 }
 
-func createProgressBar(max int, description string) *progressbar.ProgressBar {
-	return progressbar.NewOptions(max,
+func createProgressBar(max int, description string, maxDetailRow int) *progressbar.ProgressBar {
+	opts := []progressbar.Option{
 		progressbar.OptionSetDescription(description),
 		progressbar.OptionSetWidth(constants.ProgressBarWidth),
-		progressbar.OptionShowCount(),
+		progressbar.OptionSetWriter(os.Stdout),
+		progressbar.OptionSetMaxDetailRow(maxDetailRow),
 		progressbar.OptionSetTheme(progressbar.Theme{
 			Saucer:        constants.Saucer,
 			SaucerPadding: constants.SaucerPadding,
 			BarStart:      constants.BarStart,
 			BarEnd:        constants.BarEnd,
 		}),
 		progressbar.OptionThrottle(constants.ThrottleDuration),
-		progressbar.OptionClearOnFinish(),
-	)
+		progressbar.OptionOnCompletion(func() {
+			fmt.Println()
+		}),
+	}
+
+	if max > 0 {
+		opts = append(opts, progressbar.OptionShowCount())
+	} else {
+		opts = append(opts, progressbar.OptionSpinnerType(constants.SpinnerType))
+	}
+
+	return progressbar.NewOptions(max, opts...)
 }
 
 // isPodReady checks if a pod is ready
 func isPodReady(pod *corev1.Pod) bool {
+	if pod.DeletionTimestamp != nil {
+		return false
+	}
+
 	for _, condition := range pod.Status.Conditions {
-		if condition.Type == corev1.PodReady {
-			return condition.Status == corev1.ConditionTrue
+		if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
+			for _, container := range pod.Status.ContainerStatuses {
+				if !container.Ready {
+					return false
+				}
+			}
+			return true
 		}
 	}
+
 	return false
 }
 
+// getNotReadyNames
+func getNotReadyNames(pods []corev1.Pod) []string {
+	var names []string
+	for _, pod := range pods {
+		if !isPodReady(&pod) {
+			names = append(names, pod.Name)
+		}
+	}
+	return names
+}
+
 // namespaceIsActive checks if a namespace exists and is active
 func namespaceIsActive(ctx context.Context, k8sClient *client.Client, namespace string) (bool, error) {
 	ns, err := k8sClient.Clientset().CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
@@ -229,6 +281,8 @@ func waitForDeploymentReady(ctx context.Context, k8sClient *client.Client, names
 	defer cancel()
 
 	var lastDeployment *appsv1.Deployment
+	var bar *progressbar.ProgressBar
+
 	for {
 		select {
 		case <-ticker.C:
@@ -240,56 +294,92 @@ func waitForDeploymentReady(ctx context.Context, k8sClient *client.Client, names
 
 			lastDeployment = deployment
 
-			if deployment.Status.ObservedGeneration >= deployment.Generation &&
-				deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas &&
-				deployment.Status.AvailableReplicas == *deployment.Spec.Replicas &&
-				deployment.Status.ReadyReplicas == *deployment.Spec.Replicas &&
-				deployment.Status.Replicas == *deployment.Spec.Replicas {
+			desired := int(*deployment.Spec.Replicas)
+			updated := int(deployment.Status.UpdatedReplicas)
+			ready := int(deployment.Status.ReadyReplicas)
+			available := int(deployment.Status.AvailableReplicas)
+			total := int(deployment.Status.Replicas)
+			observedGen := deployment.Status.ObservedGeneration
+			gen := deployment.Generation
 
-				logger.Info("Deployment %s is ready with %d replicas", deploymentName, *deployment.Spec.Replicas)
+			if bar == nil && desired > 0 {
+				bar = createProgressBar(-1, fmt.Sprintf("Waiting for %s deployment rollout", deploymentName), 0)
+			}
+
+			if bar != nil {
+				_ = bar.Add(1)
+			}
+
+			if observedGen >= gen &&
+				updated == desired &&
+				available == desired &&
+				ready == desired &&
+				total == desired {
+				if bar != nil {
+					if err := bar.Finish(); err != nil {
+						return fmt.Errorf("finishing progress bar: %w", err)
+					}
+				}
+				logger.Info("Deployment %s is ready with %d replicas", deploymentName, desired)
 				return nil
 			}
 
-			logger.Debug("Deployment %s: %d/%d replicas ready, %d total (generation: %d/%d)",
+			logger.Debug("Deployment %s: %d/%d updated, %d/%d ready, %d total (generation: %d/%d)",
 				deploymentName,
-				deployment.Status.ReadyReplicas,
-				*deployment.Spec.Replicas,
-				deployment.Status.Replicas,
-				deployment.Status.ObservedGeneration,
-				deployment.Generation)
+				updated, desired,
+				ready, desired,
+				total,
+				observedGen, gen)
 
 		case <-timeoutCtx.Done():
+			if bar != nil {
+				if err := bar.Finish(); err != nil {
+					return fmt.Errorf("finishing progress bar: %w", err)
+				}
+			}
 			logger.Warn("Timeout reached. Deployment status:")
 			if lastDeployment != nil {
 				printDeploymentStatus(namespace, deploymentName, lastDeployment)
 			}
 
-			return fmt.Errorf("timeout waiting for deployment %s to become ready", deploymentName)
+			return fmt.Errorf("timeout waiting for deployment %s rollout", deploymentName)
 		}
 	}
 }
 
 // waitForNamespaceDeletion waits for a namespace to be completely deleted
 func waitForNamespaceDeletion(ctx context.Context, k8sClient *client.Client, namespace string, timeout time.Duration) error {
 	clientset := k8sClient.Clientset()
-
 	ticker := time.NewTicker(constants.TickerDuration)
 	defer ticker.Stop()
 
 	timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
+	bar := createProgressBar(-1, fmt.Sprintf("Stopping %s", namespace), 0)
+
 	for {
 		select {
 		case <-ticker.C:
+			_ = bar.Add(1)
 			_, err := clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
 			if err != nil {
+				if !errors.IsNotFound(err) {
+					logger.Warn("Error checking namespace: %v", err)
+					continue
+				}
+				if err := bar.Finish(); err != nil {
+					return fmt.Errorf("finishing progress bar: %w", err)
+				}
 				logger.Debug("Namespace %s successfully deleted", namespace)
 				return nil
 			}
 			logger.Debug("Namespace %s still terminating...", namespace)
 
 		case <-timeoutCtx.Done():
+			if err := bar.Finish(); err != nil {
+				return fmt.Errorf("finishing progress bar: %w", err)
+			}
 			return fmt.Errorf("timeout waiting for namespace %s to be deleted", namespace)
 		}
 	}

internal/k8s/actions/scale.go

@@ -74,13 +74,12 @@ func ScaleCmd() *cobra.Command {
 			return nil
 		}
 
-		logger.Info("Waiting for deployment to become ready...")
 		// Wait for the specific deployment (OpenSlides service name is deployment name)
 		if err := waitForDeploymentReady(ctx, k8sClient, namespace, *service, *timeout); err != nil {
 			return fmt.Errorf("waiting for deployment ready: %w", err)
 		}
 
-		logger.Info("Service scaled successfully")
+		logger.Info("%s service scaled successfully", *service)
 		return nil
 	}
 

internal/k8s/actions/start.go

@@ -76,7 +76,6 @@ func StartCmd() *cobra.Command {
 			return nil
 		}
 
-		logger.Info("Waiting for instance to become ready...")
 		if err := waitForInstanceHealthy(ctx, k8sClient, namespace, *timeout); err != nil {
 			return fmt.Errorf("waiting for ready: %w", err)
 		}

internal/k8s/actions/update_instance.go

@@ -76,7 +76,6 @@ func UpdateInstanceCmd() *cobra.Command {
 			return nil
 		}
 
-		logger.Info("Waiting for instance to become ready...")
 		if err := waitForInstanceHealthy(ctx, k8sClient, namespace, *timeout); err != nil {
 			return fmt.Errorf("waiting for instance health: %w", err)
 		}