From 04a0758b57181d017e419bcc03ff0492c76e7066 Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 24 Feb 2026 14:35:07 +0100 Subject: [PATCH 1/3] sot nop upgrade sequence, allow explicit upgrade sequence with tags --- book/src/framework/compat.md | 51 +++++++++++-- framework/.changeset/v0.14.6.md | 2 + framework/cmd/main.go | 116 ++++++++++++++++++++++------- framework/compat.go | 128 ++++++++++++-------------------- framework/compat_test.go | 2 +- 5 files changed, 189 insertions(+), 110 deletions(-) create mode 100644 framework/.changeset/v0.14.6.md diff --git a/book/src/framework/compat.md b/book/src/framework/compat.md index 4ed2bf93c..828f213fa 100644 --- a/book/src/framework/compat.md +++ b/book/src/framework/compat.md @@ -1,17 +1,58 @@ # Compatibility Testing -We have a simple tool to check compatibility for CL nodes. The example command will filter and sort the available tags, rollback and install the oldest version, and then begin performing automatic upgrades to verify that each subsequent version remains compatible with the previous one. +## Prerequisites + +Authorize in our SDLC ECR registry first. Get the creds and run +```bash +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin +``` + +## Testing Upgrade Sequence + +We have a simple tool to check compatibility for CL node clusters. The example command will filter and sort the available tags, rollback and install the oldest version, and then begin performing automatic upgrades to verify that each subsequent version remains compatible with the previous one. + +`buildcmd`, `envcmd`, `testcmd` can be arbitrary bash commands. ```bash ctf compat backward \ +--registry \ +--buildcmd "just cli" \ +--envcmd "cl r" \ +--testcmd "cl test ocr2 TestSmoke/rounds" \ +--refs 2.32.0 \ +--refs 2.33.0 \ +--refs 2.34.0 \ +--refs 2.35.0 \ +--nodes 3 +``` + +Keep in mind that `refs` should be present in regsitry you are testing against, the first (oldest) `ref` should also have a valid end-to-end test that works. + +In CI we detect SemVer tags automatically, whenever a new tag appears we select last 3, rollback to the oldest and perform upgrade process. + +```bash +ctf compat backward \ +--registry \ --buildcmd "just cli" \ --envcmd "cl r" \ --testcmd "cl test ocr2 TestSmoke/rounds" \ ---include_tags +compat \ --nodes 3 \ ---versions_back 3 +--versions-back 3 ``` -Since some of our products have a different release and tagging strategies you should add `+compat` tags to all released versions and use this tool in CI to check compatibility on `+compat` tag. +In case you have multiple DONs in your product and names of nodes are different please use `--node-name-template custom-cl-node-%d` option -Use `ctf compat restore` to rollback to current branch (default is `develop`) +## Modelling Node Operators Cluster + +It is possible to fetch versions node operators are currently running and model DON upgrade sequence locally. Logic is the same, get all the versions, rollback to the oldest one, setup product, verify, try to upgrade all the versions running the oldest test for each upgrade. + +```bash +ctf compat backward \ +--registry \ +--buildcmd "just cli" \ +--envcmd "cl r" \ +--testcmd "cl test ocr2 TestSmoke/rounds" \ +--nop northwestnodes \ +--versions-back 3 \ +--nodes 3 +``` diff --git a/framework/.changeset/v0.14.6.md b/framework/.changeset/v0.14.6.md new file mode 100644 index 000000000..9551e6ef7 --- /dev/null +++ b/framework/.changeset/v0.14.6.md @@ -0,0 +1,2 @@ +- NOPs from SOT upgrade sequence +- Allow specify upgrade sequence explicitly \ No newline at end of file diff --git a/framework/cmd/main.go b/framework/cmd/main.go index 92c8e9c4b..baca936f7 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -7,7 +7,6 @@ import ( "os/exec" "path/filepath" "runtime" - "slices" "strings" "github.com/pelletier/go-toml" @@ -275,7 +274,7 @@ Be aware that any TODO requires your attention before your run the final test! }, Usage: "Restores back to develop", Action: func(c *cli.Context) error { - return framework.RestoreToBranch(c.String("base_branch")) + return framework.CheckOut(c.String("base_branch")) }, }, { @@ -283,7 +282,7 @@ Be aware that any TODO requires your attention before your run the final test! Aliases: []string{"b"}, Flags: []cli.Flag{ &cli.IntFlag{ - Name: "versions_back", + Name: "versions-back", Aliases: []string{"v"}, Usage: "How many versions back to test", Value: 1, @@ -300,6 +299,10 @@ Be aware that any TODO requires your attention before your run the final test! Usage: "Docker Image registry for Chainlink node", Value: "smartcontract/chainlink", }, + &cli.StringFlag{ + Name: "strip-image-suffix", + Usage: "Stripts image suffix from ref to map it to registry images", + }, &cli.StringFlag{ Name: "buildcmd", Aliases: []string{"b"}, @@ -316,41 +319,102 @@ Be aware that any TODO requires your attention before your run the final test! Aliases: []string{"t"}, Usage: "Test verification command", }, + &cli.StringFlag{ + Name: "nop", + Usage: "Find specific NOPs ref upgrade sequence", + }, + &cli.StringFlag{ + Name: "node-name-template", + Usage: "CL node Docker container name template", + Value: "don-node%d", + }, + &cli.StringFlag{ + Name: "sot-url", + Usage: "RANE SOT snapshot API URL", + Value: "https://rane-sot-app.main.prod.cldev.sh/v1/snapshot", + }, + &cli.StringSliceFlag{ + Name: "refs", + Usage: "Refs to test, can be tag or commit. The corresponding image with ref should be present in registry", + }, &cli.StringSliceFlag{ - Name: "include_tags", - Usage: "Patterns to include specific tags (e.g., beta,rc,v0,v1)", + Name: "include-refs", + Usage: "Patterns to include specific refs (e.g., beta,rc,v0,v1), also used in CI to verify on test refs(tags)", }, &cli.StringSliceFlag{ - Name: "exclude_tags", - Usage: "Patterns to exclude specific tags (e.g., beta,rc,v0,v1)", - Value: cli.NewStringSlice("beta", "rc", "v0", "ccip", "cre", "datastreams"), + Name: "exclude-refs", + Usage: "Patterns to exclude specific refs (e.g., beta,rc,v0,v1)", + // Value: cli.NewStringSlice("beta", "rc", "v0", "ccip", "cre", "datastreams", "streams"), }, }, Usage: "Rollbacks N versions back, runs the test the upgrades CL nodes with new versions", Action: func(c *cli.Context) error { - versionsBack := c.Int("versions_back") + versionsBack := c.Int("versions-back") registry := c.String("registry") - include := c.StringSlice("include_tags") - exclude := c.StringSlice("exclude_tags") + refs := c.StringSlice("refs") + include := c.StringSlice("include-refs") + exclude := c.StringSlice("exclude-refs") buildcmd := c.String("buildcmd") envcmd := c.String("envcmd") testcmd := c.String("testcmd") nodes := c.Int("nodes") + nodeNameTemplate := c.String("node-name-template") + + // this param is purely for testing if old image versions + // do not have test or image to verify in some registries + testImgSuffix := c.String("strip-image-suffix") + + nop := c.String("nop") + sotURL := c.String("sot-url") + // test logic is: - // - rollback to selected tag + // - rollback to selected ref // - spin up the env and perform the initial smoke test // - upgrade N CL nodes with preversing DB volume (shared database) // - perform the test again // - repeat until all the new versions are validated - tags, err := framework.RollbackToEarliestSemverTag(versionsBack, include, exclude) - if err != nil { - return err + + // if no refs provided find refs (tags) sequence SemVer sequence for last N versions_back + // else, use refs param slice + // + var err error + if len(refs) == 0 && nop == "" { + refs, err = framework.FindSemVerRefSequence(versionsBack, include, exclude) + if err != nil { + return err + } + } + if nop != "" { + refs, err = framework.FindNOPRefs(sotURL, nop, exclude) + if err != nil { + return err + } } - if envcmd == "" || testcmd == "" { + framework.L.Info(). + Int("TotalSemVerRefs", len(refs)). + Strs("SelectedRefs", refs). + Str("EarliestRefs", refs[0]). + Msg("Formed upgrade sequence") + // if no commands just show the tags and return + if buildcmd == "" || envcmd == "" || testcmd == "" { framework.L.Info().Msg("No envcmd or testcmd provided, skipping") return nil } + // checkout the oldest ref + if err := framework.CheckOut(refs[0]); err != nil { + return err + } + + // if we know there are no tests to verify specific version map to test refs + // by stripping suffix + for i := range refs { + refs[i] = strings.ReplaceAll(refs[i], testImgSuffix, "") + } + + // setup the env and verify with test command + framework.L.Info().Strs("Sequence", refs).Msg("Running upgrade sequence") + os.Setenv("CHAINLINK_IMAGE", fmt.Sprintf("%s:%s", registry, refs[0])) if _, err := framework.ExecCmdWithContext(c.Context, buildcmd); err != nil { return err } @@ -360,16 +424,18 @@ Be aware that any TODO requires your attention before your run the final test! if _, err := framework.ExecCmdWithContext(c.Context, testcmd); err != nil { return err } - // reverse and skip current version - slices.Reverse(tags) - tags = tags[1:] - for _, tag := range tags { - tagToPull := strings.ReplaceAll(tag, "+compat", "") + // start upgrading nodes and verifying with tests, all the versions except the oldest one + for _, tag := range refs[1:] { + framework.L.Info(). + Int("Nodes", nodes). + Str("Version", tag). + Msg("Upgrading nodes") + img := fmt.Sprintf("%s:%s", registry, tag) + if _, err := framework.ExecCmdWithContext(c.Context, fmt.Sprintf("docker pull %s", img)); err != nil { + return fmt.Errorf("failed to pull image %s: %w", img, err) + } for i := range nodes { - if err := framework.UpgradeContainer( - c.Context, - fmt.Sprintf("don-node%d", i), - fmt.Sprintf("%s:%s", registry, tagToPull)); err != nil { + if err := framework.UpgradeContainer(c.Context, fmt.Sprintf(nodeNameTemplate, i), img); err != nil { return err } } diff --git a/framework/compat.go b/framework/compat.go index 05a040004..9266e2f45 100644 --- a/framework/compat.go +++ b/framework/compat.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "slices" "sort" "strings" @@ -27,7 +28,7 @@ func UpgradeContainer(ctx context.Context, containerName, newImage string) error Str("Container", containerName). Str("Image", newImage). Logger() - l.Info().Msg("Upgrading container") + l.Debug().Msg("Upgrading container") cli, err := client.NewClientWithOpts( client.FromEnv, client.WithAPIVersionNegotiation(), @@ -51,11 +52,6 @@ func UpgradeContainer(ctx context.Context, containerName, newImage string) error if err := cli.ContainerRemove(ctx, containerName, removeOpts); err != nil { return fmt.Errorf("failed to remove container %s: %w", containerName, err) } - l.Debug().Msg("Pulling new image") - if _, err := ExecCmdWithContext(ctx, fmt.Sprintf("docker pull %s", newImage)); err != nil { - return fmt.Errorf("failed to pull image %s: %w", newImage, err) - } - l.Debug().Msg("Image pulled successfully") inspect.Config.Image = newImage @@ -85,27 +81,15 @@ func UpgradeContainer(ctx context.Context, containerName, newImage string) error if err := cli.ContainerStart(ctx, createResp.ID, startOpts); err != nil { return fmt.Errorf("failed to start container %s: %w", containerName, err) } - l.Info(). + l.Debug(). Str("ContainerID", createResp.ID[:12]). Msg("Container successfully rebooted with new image") return nil } -// RestoreToBranch restores git back to the develop branch -func RestoreToBranch(baseBranch string) error { - _, err := ExecCmd(fmt.Sprintf("git checkout %s", baseBranch)) - if err != nil { - return fmt.Errorf("failed to checkout develop branch: %w", err) - } - L.Info(). - Str("Branch", "develop"). - Msg("Successfully restored to develop branch") - return nil -} - -// RollbackToEarliestSemverTag gets all semver tags, sorts them, and rolls back to the earliest tag +// FindSemVerRefSequence gets all semver tags, sorts them, and rolls back to the earliest tag // returns all the tags starting from the oldest one -func RollbackToEarliestSemverTag(tagsBack int, include, exclude []string) ([]string, error) { +func FindSemVerRefSequence(tagsBack int, include, exclude []string) ([]string, error) { output, err := ExecCmd("git tag --list") if err != nil { return nil, fmt.Errorf("failed to list git tags: %w", err) @@ -116,7 +100,7 @@ func RollbackToEarliestSemverTag(tagsBack int, include, exclude []string) ([]str return nil, fmt.Errorf("no tags found in repository") } - sortedDesc := SortSemverTags(tags, include, exclude) + sortedDesc := FilterSemverTags(tags, include, exclude) if len(sortedDesc) == 0 { return nil, fmt.Errorf("no valid semver tags found") } @@ -125,27 +109,20 @@ func RollbackToEarliestSemverTag(tagsBack int, include, exclude []string) ([]str if len(sortedDesc) > tagsBack { remainingTags = sortedDesc[:tagsBack] } - earliestTag := remainingTags[len(remainingTags)-1] - L.Info(). - Int("TotalValidTags", len(sortedDesc)). - Strs("SelectedTags", remainingTags). - Str("EarliestTag", earliestTag). - Msg("Selected previous tag") + slices.Reverse(remainingTags) + return remainingTags, nil +} - _, err = ExecCmd("git checkout " + earliestTag) +func CheckOut(ref string) error { + _, err := ExecCmd("git checkout " + ref) if err != nil { - L.Error(). - Str("Tag", earliestTag). - Err(err). - Msg("Failed to checkout tag") - return nil, fmt.Errorf("failed to checkout tag %s: %w", earliestTag, err) + return fmt.Errorf("failed to checkout ref %s: %w", ref, err) } - L.Info(). - Str("Tag", earliestTag). - Msg("Successfully rolled back to tag") - return remainingTags, nil + Str("Ref", ref). + Msg("Successfully rolled back to ref") + return nil } type RaneSOTResponseBody struct { @@ -155,17 +132,16 @@ type RaneSOTResponseBody struct { } `json:"nodes"` } -// GetTagsFromURL fetches tags from a JSON endpoint and applies filtering -func GetTagsFromURL(url, imageTagSuffix string, nopSuffixes, ignores []string) ([]string, error) { +// FindNOPRefs fetches NOP tags from a RANE SOT source +func FindNOPRefs(url string, nopName string, exclude []string) ([]string, error) { L.Info(). Str("URL", url). - Str("ImageTagSuffix", imageTagSuffix). - Strs("NOPs", nopSuffixes). - Strs("IgnoreSuffix", ignores). - Msg("Fetching tags from snapshot") + Str("NOP", nopName). + Strs("Exclude", exclude). + Msg("Fetching refs from snapshot") resp, err := http.Get(url) //nolint:gosec if err != nil { - return nil, fmt.Errorf("failed to fetch URL: %w", err) + return nil, fmt.Errorf("failed to fetch SOT URL: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { @@ -173,54 +149,50 @@ func GetTagsFromURL(url, imageTagSuffix string, nopSuffixes, ignores []string) ( } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + return nil, fmt.Errorf("failed to read SOT URL response body: %w", err) } var response RaneSOTResponseBody if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("failed to parse JSON: %w", err) + return nil, fmt.Errorf("failed to parse SOT response body: %w", err) } - tags := make([]string, 0) - seenTags := make(map[string]bool, 0) + refs := make([]string, 0) + nops := make([]string, 0) + seenRefs := make(map[string]bool, 0) + seenNOPs := make(map[string]bool, 0) // check all the nodes for uniq tags for _, node := range response.Nodes { version := node.Version nop := node.NOP - // skip if we are not interested in this NOP - for _, ns := range nopSuffixes { - if !strings.Contains(nop, ns) { - continue - } - } + // add uniq NOP + if _, ok := seenNOPs[nop]; !ok { + nops = append(nops, nop) + seenNOPs[nop] = true + } + // continue if it's not the NOP we need + if nop != nopName { + continue + } // skip if version is empty if version == "" { continue } - - // skip if we ignore some images - ignored := false - for _, ignore := range ignores { - if strings.Contains(version, ignore) { - ignored = true - break - } - } - - if strings.Contains(version, imageTagSuffix) && !ignored { - if _, ok := seenTags[version]; ok { - continue - } - tags = append(tags, version) - seenTags[version] = true + // add uniq version + if _, ok := seenRefs[version]; !ok { + refs = append(refs, version) + seenRefs[version] = true } } - return tags, nil + semverTags := FilterSemverTags(refs, []string{}, exclude) + slices.Reverse(semverTags) + L.Info().Strs("NOPs", nops).Msg("Scanned NOPs") + return semverTags, nil } -// SortSemverTags parses valid versions and returns them sorted from latest to lowest -func SortSemverTags(versions []string, include []string, exclude []string) []string { +// FilterSemverTags parses valid versions and returns them sorted from latest to lowest +func FilterSemverTags(versions []string, include []string, exclude []string) []string { if len(versions) == 0 { return []string{} } @@ -234,17 +206,15 @@ func SortSemverTags(versions []string, include []string, exclude []string) []str } parsedVersions = append(parsedVersions, parsed) } - // descending order sort.Slice(parsedVersions, func(i, j int) bool { return parsedVersions[i].GreaterThan(parsedVersions[j]) }) - // fliter include/exclude filtered := filterVersions(parsedVersions, include, exclude) L.Info(). - Strs("include", include). - Strs("exclude", exclude). + Strs("Include", include). + Strs("Exclude", exclude). Msg("Applied filters") return filtered } @@ -278,4 +248,4 @@ func matchesFilter(version string, include, exclude []string) bool { } } return false -} \ No newline at end of file +} diff --git a/framework/compat_test.go b/framework/compat_test.go index b15bc0df3..95b7fea58 100644 --- a/framework/compat_test.go +++ b/framework/compat_test.go @@ -97,7 +97,7 @@ func TestSmokeTagsFilter(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - tags := SortSemverTags(tc.tags, tc.includes, tc.excludes) + tags := FilterSemverTags(tc.tags, tc.includes, tc.excludes) require.Equal(t, tc.expectedTags, tags, "Test case: %s", tc.name) }) } From de0581d3551120846dd3c9204101438540fe51ae Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 24 Feb 2026 14:41:46 +0100 Subject: [PATCH 2/3] simplify comments --- framework/cmd/main.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/framework/cmd/main.go b/framework/cmd/main.go index baca936f7..2733a1a50 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -361,10 +361,6 @@ Be aware that any TODO requires your attention before your run the final test! nodes := c.Int("nodes") nodeNameTemplate := c.String("node-name-template") - // this param is purely for testing if old image versions - // do not have test or image to verify in some registries - testImgSuffix := c.String("strip-image-suffix") - nop := c.String("nop") sotURL := c.String("sot-url") @@ -406,8 +402,9 @@ Be aware that any TODO requires your attention before your run the final test! return err } - // if we know there are no tests to verify specific version map to test refs - // by stripping suffix + // this is a hack allowing us to match what we have in Git to registry or NOP version + // it'd exist until we stabilize tagging strategy + testImgSuffix := c.String("strip-image-suffix") for i := range refs { refs[i] = strings.ReplaceAll(refs[i], testImgSuffix, "") } From 37e7f99def3df78c97158db93c2a0e35bed1ed8e Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 24 Feb 2026 15:44:53 +0100 Subject: [PATCH 3/3] clean up excludes --- framework/cmd/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/framework/cmd/main.go b/framework/cmd/main.go index 2733a1a50..c58096b92 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -344,7 +344,6 @@ Be aware that any TODO requires your attention before your run the final test! &cli.StringSliceFlag{ Name: "exclude-refs", Usage: "Patterns to exclude specific refs (e.g., beta,rc,v0,v1)", - // Value: cli.NewStringSlice("beta", "rc", "v0", "ccip", "cre", "datastreams", "streams"), }, }, Usage: "Rollbacks N versions back, runs the test the upgrades CL nodes with new versions",