Skip to content

[wip]OTA-1813: Extend alert risk e2e #1371

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hongkailiu wants to merge 1 commit into
base: main
Choose a base branch
from
+83 −10
Open

[wip]OTA-1813: Extend alert risk e2e #1371

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 83 additions & 10 deletions test/cvo/accept_risks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
o "github.com/onsi/gomega"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
prometheusoperatorv1client "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
"sigs.k8s.io/yaml"

"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand Down Expand Up @@ -38,6 +39,11 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
backup configv1.ClusterVersionSpec
)

const (
prometheusRuleForTesting = "testing"
alertNameForTesting = "TestAlertFeatureE2ETestOTA1813"
)

g.BeforeEach(func() {
c, err = util.GetRestConfig()
o.Expect(err).To(o.BeNil())
Expand All @@ -60,6 +66,10 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
})

g.AfterEach(func() {
err := monitoringClient.PrometheusRules(external.DefaultCVONamespace).Delete(ctx, prometheusRuleForTesting, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
o.Expect(err).To(o.BeNil())
}
if needRecover {
cv, err := configClient.ClusterVersions().Get(ctx, external.DefaultClusterVersionName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
Expand All @@ -78,7 +88,7 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,

g.By("Using fauxinnati as the upstream and its simple channel")
cv.Spec.Upstream = util.FauxinnatiAPIURL
cv.Spec.Channel = "simple"
cv.Spec.Channel = "OTA-1813"
Copy link
Copy Markdown
Member Author

@hongkailiu hongkailiu Apr 10, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The channel will return something like this:

$ curl "https://fauxinnati-fauxinnati.apps.ota-stage.q2z4.p1.openshiftapps.com/api/upgrades_info/graph?channel=OTA-1813&version=4.22.0-0.nightly-2026-04-06-051707&arch=amd64"
{
  "nodes": [
    {
      "version": "4.22.0-0.nightly-2026-04-06-051707",
      "payload": "quay.io/openshift-release-dev/ocp-release@sha256:00000000000000000000000000000000000000000000000000000000003d5ef0",
      "metadata": {
        "io.openshift.upgrades.graph.release.channels": "OTA-1813",
        "io.openshift.upgrades.graph.release.manifestref": "sha256:00000000000000000000000000000000000000000000000000000000003d5ef0",
        "url": "https://access.redhat.com/errata/RHSA-2024:06200"
      }
    },
    {
      "version": "4.22.1",
      "payload": "quay.io/openshift-release-dev/ocp-release@sha256:00000000000000000000000000000000000000000000000000000000003d5ef1",
      "metadata": {
        "io.openshift.upgrades.graph.release.channels": "OTA-1813",
        "io.openshift.upgrades.graph.release.manifestref": "sha256:00000000000000000000000000000000000000000000000000000000003d5ef1",
        "url": "https://access.redhat.com/errata/RHSA-2024:06201"
      }
    },
    {
      "version": "4.22.2",
      "payload": "quay.io/openshift-release-dev/ocp-release@sha256:00000000000000000000000000000000000000000000000000000000003d5ef2",
      "metadata": {
        "io.openshift.upgrades.graph.release.channels": "OTA-1813",
        "io.openshift.upgrades.graph.release.manifestref": "sha256:00000000000000000000000000000000000000000000000000000000003d5ef2",
        "url": "https://access.redhat.com/errata/RHSA-2024:06202"
      }
    }
  ],
  "edges": [
    [
      0,
      1
    ]
  ],
  "conditionalEdges": [
    {
      "edges": [
        {
          "from": "4.22.0-0.nightly-2026-04-06-051707",
          "to": "4.22.2"
        }
      ],
      "risks": [
        {
          "url": "https://docs.openshift.com/synthetic-risk-a",
          "name": "SomeInvokerThing",
          "message": "This is SomeInvokerThing that cannot be evaluated for testing purposes",
          "matchingRules": [
            {
              "type": "PromQL",
              "promql": {
                "promql": "this will fail; muahaha"
              }
            }
          ]
        }
      ]
    }
  ]
}


_, err = configClient.ClusterVersions().Update(ctx, cv, metav1.UpdateOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
Expand All @@ -87,7 +97,7 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
g.By("Create a critical alert for testing")
prometheusRule := &monitoringv1.PrometheusRule{
ObjectMeta: metav1.ObjectMeta{
Name: "testing",
Name: prometheusRuleForTesting,
Namespace: external.DefaultCVONamespace,
},
Spec: monitoringv1.PrometheusRuleSpec{
Expand All @@ -96,7 +106,7 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
Name: "test",
Rules: []monitoringv1.Rule{
{
Alert: "TestAlertFeatureE2ETestOTA1813",
Alert: alertNameForTesting,
Annotations: map[string]string{"summary": "Test summary.", "description": "Test description."},
Expr: intstr.IntOrString{
Type: intstr.String,
Expand All @@ -111,19 +121,13 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
}
created, err := monitoringClient.PrometheusRules(external.DefaultCVONamespace).Create(ctx, prometheusRule, metav1.CreateOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
defer func() {
err := monitoringClient.PrometheusRules(external.DefaultCVONamespace).Delete(ctx, created.Name, metav1.DeleteOptions{})
if !errors.IsNotFound(err) {
o.Expect(err).To(o.BeNil())
}
}()

g.By("Checking if the risk shows up in ClusterVersion's status")
o.Expect(wait.PollUntilContextTimeout(ctx, 30*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) {
cv, err := configClient.ClusterVersions().Get(ctx, external.DefaultClusterVersionName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
for _, risk := range cv.Status.ConditionalUpdateRisks {
if risk.Name == "TestAlertFeatureE2ETestOTA1813" {
if risk.Name == alertNameForTesting {
if c := meta.FindStatusCondition(risk.Conditions, external.ConditionalUpdateRiskConditionTypeApplies); c != nil {
if c.Status == metav1.ConditionTrue && external.IsAlertConditionReason(c.Reason) {
return true, nil
Expand All @@ -149,6 +153,66 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
}
return true, nil
})).NotTo(o.HaveOccurred(), "still recommending updates while alert is firing")

g.By("Checking that there are recommended conditional updates promoted to available updates after the risk from alert is accepted")
cv, err = configClient.ClusterVersions().Get(ctx, external.DefaultClusterVersionName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
if cv.Spec.DesiredUpdate == nil {
cv.Spec.DesiredUpdate = &configv1.Update{}
}
cv.Spec.DesiredUpdate.AcceptRisks = append(cv.Spec.DesiredUpdate.AcceptRisks, configv1.AcceptRisk{Name: alertNameForTesting})
_, err = configClient.ClusterVersions().Update(ctx, cv, metav1.UpdateOptions{})
o.Expect(err).NotTo(o.HaveOccurred())

o.Expect(wait.PollUntilContextTimeout(ctx, 30*time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) {
cv, err := configClient.ClusterVersions().Get(ctx, external.DefaultClusterVersionName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
if len(cv.Status.AvailableUpdates) == 0 {
return false, nil
}
for _, cu := range cv.Status.ConditionalUpdates {
condition := meta.FindStatusCondition(cu.Conditions, external.ConditionalUpdateConditionTypeRecommended)
if condition != nil && condition.Status == metav1.ConditionTrue {
return true, nil
}
}
return false, nil
})).NotTo(o.HaveOccurred(), "still no recommended conditional updates after the risk from alert is accepted")

g.By("Checking that there are no duplicated versions in conditional updates")
cv, err = configClient.ClusterVersions().Get(ctx, external.DefaultClusterVersionName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
versions := sets.New[string]()
for _, cu := range cv.Status.ConditionalUpdates {
o.Expect(versions.Has(cu.Release.Version)).To(o.BeFalse(), "ConditionalUpdates.Release.Version %s is duplicated. cv/version is:\n%s", cu.Release.Version, getYaml(*cv))
versions.Insert(cu.Release.Version)
}

g.By("Checking that there are no duplicated versions in available updates")
versions = sets.New[string]()
for _, u := range cv.Status.AvailableUpdates {
o.Expect(versions.Has(u.Version)).To(o.BeFalse(), "AvailableUpdates.Version %s is duplicated. cv/version is:\n%s", u.Version, getYaml(*cv))
versions.Insert(u.Version)
}

g.By("Checking that there are no recommended conditional updates after the alert is resolved")
err = monitoringClient.PrometheusRules(external.DefaultCVONamespace).Delete(ctx, created.Name, metav1.DeleteOptions{})
o.Expect(err).NotTo(o.HaveOccurred())

o.Expect(wait.PollUntilContextTimeout(ctx, 30*time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) {
cv, err := configClient.ClusterVersions().Get(ctx, external.DefaultClusterVersionName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
for _, cu := range cv.Status.ConditionalUpdates {
condition := meta.FindStatusCondition(cu.Conditions, external.ConditionalUpdateConditionTypeRecommended)
if condition != nil && condition.Status == metav1.ConditionTrue {
return false, nil
}
}
return true, nil
})).NotTo(o.HaveOccurred(), "still recommended conditional updates after the alert is resolved")

g.By("Checking that there are available updates after the alert is resolved")
o.Expect(cv.Status.AvailableUpdates).NotTo(o.BeEmpty())
})

g.It("should work with accept risks", g.Label("Serial"), func() {
Expand Down Expand Up @@ -239,3 +303,12 @@ var _ = g.Describe(`[Jira:"Cluster Version Operator"] cluster-version-operator`,
}
})
})

func getYaml(cv configv1.ClusterVersion) string {
raw, err := yaml.Marshal(cv)
if err != nil {
logger.Error(err, "failed to marshal ClusterVersion")
return ""
}
return string(raw)
}