diff --git a/book/src/framework/compat.md b/book/src/framework/compat.md index 4ed2bf93c..828f213fa 100644 --- a/book/src/framework/compat.md +++ b/book/src/framework/compat.md @@ -1,17 +1,58 @@ # Compatibility Testing -We have a simple tool to check compatibility for CL nodes. The example command will filter and sort the available tags, rollback and install the oldest version, and then begin performing automatic upgrades to verify that each subsequent version remains compatible with the previous one. +## Prerequisites + +Authorize in our SDLC ECR registry first. Get the creds and run +```bash +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin +``` + +## Testing Upgrade Sequence + +We have a simple tool to check compatibility for CL node clusters. The example command will filter and sort the available tags, rollback and install the oldest version, and then begin performing automatic upgrades to verify that each subsequent version remains compatible with the previous one. + +`buildcmd`, `envcmd`, `testcmd` can be arbitrary bash commands. ```bash ctf compat backward \ +--registry \ +--buildcmd "just cli" \ +--envcmd "cl r" \ +--testcmd "cl test ocr2 TestSmoke/rounds" \ +--refs 2.32.0 \ +--refs 2.33.0 \ +--refs 2.34.0 \ +--refs 2.35.0 \ +--nodes 3 +``` + +Keep in mind that `refs` should be present in regsitry you are testing against, the first (oldest) `ref` should also have a valid end-to-end test that works. + +In CI we detect SemVer tags automatically, whenever a new tag appears we select last 3, rollback to the oldest and perform upgrade process. + +```bash +ctf compat backward \ +--registry \ --buildcmd "just cli" \ --envcmd "cl r" \ --testcmd "cl test ocr2 TestSmoke/rounds" \ ---include_tags +compat \ --nodes 3 \ ---versions_back 3 +--versions-back 3 ``` -Since some of our products have a different release and tagging strategies you should add `+compat` tags to all released versions and use this tool in CI to check compatibility on `+compat` tag. +In case you have multiple DONs in your product and names of nodes are different please use `--node-name-template custom-cl-node-%d` option -Use `ctf compat restore` to rollback to current branch (default is `develop`) +## Modelling Node Operators Cluster + +It is possible to fetch versions node operators are currently running and model DON upgrade sequence locally. Logic is the same, get all the versions, rollback to the oldest one, setup product, verify, try to upgrade all the versions running the oldest test for each upgrade. + +```bash +ctf compat backward \ +--registry \ +--buildcmd "just cli" \ +--envcmd "cl r" \ +--testcmd "cl test ocr2 TestSmoke/rounds" \ +--nop northwestnodes \ +--versions-back 3 \ +--nodes 3 +``` diff --git a/framework/.changeset/v0.14.6.md b/framework/.changeset/v0.14.6.md new file mode 100644 index 000000000..9551e6ef7 --- /dev/null +++ b/framework/.changeset/v0.14.6.md @@ -0,0 +1,2 @@ +- NOPs from SOT upgrade sequence +- Allow specify upgrade sequence explicitly \ No newline at end of file diff --git a/framework/cmd/main.go b/framework/cmd/main.go index 92c8e9c4b..c58096b92 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -7,7 +7,6 @@ import ( "os/exec" "path/filepath" "runtime" - "slices" "strings" "github.com/pelletier/go-toml" @@ -275,7 +274,7 @@ Be aware that any TODO requires your attention before your run the final test! }, Usage: "Restores back to develop", Action: func(c *cli.Context) error { - return framework.RestoreToBranch(c.String("base_branch")) + return framework.CheckOut(c.String("base_branch")) }, }, { @@ -283,7 +282,7 @@ Be aware that any TODO requires your attention before your run the final test! Aliases: []string{"b"}, Flags: []cli.Flag{ &cli.IntFlag{ - Name: "versions_back", + Name: "versions-back", Aliases: []string{"v"}, Usage: "How many versions back to test", Value: 1, @@ -300,6 +299,10 @@ Be aware that any TODO requires your attention before your run the final test! Usage: "Docker Image registry for Chainlink node", Value: "smartcontract/chainlink", }, + &cli.StringFlag{ + Name: "strip-image-suffix", + Usage: "Stripts image suffix from ref to map it to registry images", + }, &cli.StringFlag{ Name: "buildcmd", Aliases: []string{"b"}, @@ -316,41 +319,98 @@ Be aware that any TODO requires your attention before your run the final test! Aliases: []string{"t"}, Usage: "Test verification command", }, + &cli.StringFlag{ + Name: "nop", + Usage: "Find specific NOPs ref upgrade sequence", + }, + &cli.StringFlag{ + Name: "node-name-template", + Usage: "CL node Docker container name template", + Value: "don-node%d", + }, + &cli.StringFlag{ + Name: "sot-url", + Usage: "RANE SOT snapshot API URL", + Value: "https://rane-sot-app.main.prod.cldev.sh/v1/snapshot", + }, + &cli.StringSliceFlag{ + Name: "refs", + Usage: "Refs to test, can be tag or commit. The corresponding image with ref should be present in registry", + }, &cli.StringSliceFlag{ - Name: "include_tags", - Usage: "Patterns to include specific tags (e.g., beta,rc,v0,v1)", + Name: "include-refs", + Usage: "Patterns to include specific refs (e.g., beta,rc,v0,v1), also used in CI to verify on test refs(tags)", }, &cli.StringSliceFlag{ - Name: "exclude_tags", - Usage: "Patterns to exclude specific tags (e.g., beta,rc,v0,v1)", - Value: cli.NewStringSlice("beta", "rc", "v0", "ccip", "cre", "datastreams"), + Name: "exclude-refs", + Usage: "Patterns to exclude specific refs (e.g., beta,rc,v0,v1)", }, }, Usage: "Rollbacks N versions back, runs the test the upgrades CL nodes with new versions", Action: func(c *cli.Context) error { - versionsBack := c.Int("versions_back") + versionsBack := c.Int("versions-back") registry := c.String("registry") - include := c.StringSlice("include_tags") - exclude := c.StringSlice("exclude_tags") + refs := c.StringSlice("refs") + include := c.StringSlice("include-refs") + exclude := c.StringSlice("exclude-refs") buildcmd := c.String("buildcmd") envcmd := c.String("envcmd") testcmd := c.String("testcmd") nodes := c.Int("nodes") + nodeNameTemplate := c.String("node-name-template") + + nop := c.String("nop") + sotURL := c.String("sot-url") + // test logic is: - // - rollback to selected tag + // - rollback to selected ref // - spin up the env and perform the initial smoke test // - upgrade N CL nodes with preversing DB volume (shared database) // - perform the test again // - repeat until all the new versions are validated - tags, err := framework.RollbackToEarliestSemverTag(versionsBack, include, exclude) - if err != nil { - return err + + // if no refs provided find refs (tags) sequence SemVer sequence for last N versions_back + // else, use refs param slice + // + var err error + if len(refs) == 0 && nop == "" { + refs, err = framework.FindSemVerRefSequence(versionsBack, include, exclude) + if err != nil { + return err + } } - if envcmd == "" || testcmd == "" { + if nop != "" { + refs, err = framework.FindNOPRefs(sotURL, nop, exclude) + if err != nil { + return err + } + } + framework.L.Info(). + Int("TotalSemVerRefs", len(refs)). + Strs("SelectedRefs", refs). + Str("EarliestRefs", refs[0]). + Msg("Formed upgrade sequence") + // if no commands just show the tags and return + if buildcmd == "" || envcmd == "" || testcmd == "" { framework.L.Info().Msg("No envcmd or testcmd provided, skipping") return nil } + // checkout the oldest ref + if err := framework.CheckOut(refs[0]); err != nil { + return err + } + + // this is a hack allowing us to match what we have in Git to registry or NOP version + // it'd exist until we stabilize tagging strategy + testImgSuffix := c.String("strip-image-suffix") + for i := range refs { + refs[i] = strings.ReplaceAll(refs[i], testImgSuffix, "") + } + + // setup the env and verify with test command + framework.L.Info().Strs("Sequence", refs).Msg("Running upgrade sequence") + os.Setenv("CHAINLINK_IMAGE", fmt.Sprintf("%s:%s", registry, refs[0])) if _, err := framework.ExecCmdWithContext(c.Context, buildcmd); err != nil { return err } @@ -360,16 +420,18 @@ Be aware that any TODO requires your attention before your run the final test! if _, err := framework.ExecCmdWithContext(c.Context, testcmd); err != nil { return err } - // reverse and skip current version - slices.Reverse(tags) - tags = tags[1:] - for _, tag := range tags { - tagToPull := strings.ReplaceAll(tag, "+compat", "") + // start upgrading nodes and verifying with tests, all the versions except the oldest one + for _, tag := range refs[1:] { + framework.L.Info(). + Int("Nodes", nodes). + Str("Version", tag). + Msg("Upgrading nodes") + img := fmt.Sprintf("%s:%s", registry, tag) + if _, err := framework.ExecCmdWithContext(c.Context, fmt.Sprintf("docker pull %s", img)); err != nil { + return fmt.Errorf("failed to pull image %s: %w", img, err) + } for i := range nodes { - if err := framework.UpgradeContainer( - c.Context, - fmt.Sprintf("don-node%d", i), - fmt.Sprintf("%s:%s", registry, tagToPull)); err != nil { + if err := framework.UpgradeContainer(c.Context, fmt.Sprintf(nodeNameTemplate, i), img); err != nil { return err } } diff --git a/framework/compat.go b/framework/compat.go index 05a040004..9266e2f45 100644 --- a/framework/compat.go +++ b/framework/compat.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "slices" "sort" "strings" @@ -27,7 +28,7 @@ func UpgradeContainer(ctx context.Context, containerName, newImage string) error Str("Container", containerName). Str("Image", newImage). Logger() - l.Info().Msg("Upgrading container") + l.Debug().Msg("Upgrading container") cli, err := client.NewClientWithOpts( client.FromEnv, client.WithAPIVersionNegotiation(), @@ -51,11 +52,6 @@ func UpgradeContainer(ctx context.Context, containerName, newImage string) error if err := cli.ContainerRemove(ctx, containerName, removeOpts); err != nil { return fmt.Errorf("failed to remove container %s: %w", containerName, err) } - l.Debug().Msg("Pulling new image") - if _, err := ExecCmdWithContext(ctx, fmt.Sprintf("docker pull %s", newImage)); err != nil { - return fmt.Errorf("failed to pull image %s: %w", newImage, err) - } - l.Debug().Msg("Image pulled successfully") inspect.Config.Image = newImage @@ -85,27 +81,15 @@ func UpgradeContainer(ctx context.Context, containerName, newImage string) error if err := cli.ContainerStart(ctx, createResp.ID, startOpts); err != nil { return fmt.Errorf("failed to start container %s: %w", containerName, err) } - l.Info(). + l.Debug(). Str("ContainerID", createResp.ID[:12]). Msg("Container successfully rebooted with new image") return nil } -// RestoreToBranch restores git back to the develop branch -func RestoreToBranch(baseBranch string) error { - _, err := ExecCmd(fmt.Sprintf("git checkout %s", baseBranch)) - if err != nil { - return fmt.Errorf("failed to checkout develop branch: %w", err) - } - L.Info(). - Str("Branch", "develop"). - Msg("Successfully restored to develop branch") - return nil -} - -// RollbackToEarliestSemverTag gets all semver tags, sorts them, and rolls back to the earliest tag +// FindSemVerRefSequence gets all semver tags, sorts them, and rolls back to the earliest tag // returns all the tags starting from the oldest one -func RollbackToEarliestSemverTag(tagsBack int, include, exclude []string) ([]string, error) { +func FindSemVerRefSequence(tagsBack int, include, exclude []string) ([]string, error) { output, err := ExecCmd("git tag --list") if err != nil { return nil, fmt.Errorf("failed to list git tags: %w", err) @@ -116,7 +100,7 @@ func RollbackToEarliestSemverTag(tagsBack int, include, exclude []string) ([]str return nil, fmt.Errorf("no tags found in repository") } - sortedDesc := SortSemverTags(tags, include, exclude) + sortedDesc := FilterSemverTags(tags, include, exclude) if len(sortedDesc) == 0 { return nil, fmt.Errorf("no valid semver tags found") } @@ -125,27 +109,20 @@ func RollbackToEarliestSemverTag(tagsBack int, include, exclude []string) ([]str if len(sortedDesc) > tagsBack { remainingTags = sortedDesc[:tagsBack] } - earliestTag := remainingTags[len(remainingTags)-1] - L.Info(). - Int("TotalValidTags", len(sortedDesc)). - Strs("SelectedTags", remainingTags). - Str("EarliestTag", earliestTag). - Msg("Selected previous tag") + slices.Reverse(remainingTags) + return remainingTags, nil +} - _, err = ExecCmd("git checkout " + earliestTag) +func CheckOut(ref string) error { + _, err := ExecCmd("git checkout " + ref) if err != nil { - L.Error(). - Str("Tag", earliestTag). - Err(err). - Msg("Failed to checkout tag") - return nil, fmt.Errorf("failed to checkout tag %s: %w", earliestTag, err) + return fmt.Errorf("failed to checkout ref %s: %w", ref, err) } - L.Info(). - Str("Tag", earliestTag). - Msg("Successfully rolled back to tag") - return remainingTags, nil + Str("Ref", ref). + Msg("Successfully rolled back to ref") + return nil } type RaneSOTResponseBody struct { @@ -155,17 +132,16 @@ type RaneSOTResponseBody struct { } `json:"nodes"` } -// GetTagsFromURL fetches tags from a JSON endpoint and applies filtering -func GetTagsFromURL(url, imageTagSuffix string, nopSuffixes, ignores []string) ([]string, error) { +// FindNOPRefs fetches NOP tags from a RANE SOT source +func FindNOPRefs(url string, nopName string, exclude []string) ([]string, error) { L.Info(). Str("URL", url). - Str("ImageTagSuffix", imageTagSuffix). - Strs("NOPs", nopSuffixes). - Strs("IgnoreSuffix", ignores). - Msg("Fetching tags from snapshot") + Str("NOP", nopName). + Strs("Exclude", exclude). + Msg("Fetching refs from snapshot") resp, err := http.Get(url) //nolint:gosec if err != nil { - return nil, fmt.Errorf("failed to fetch URL: %w", err) + return nil, fmt.Errorf("failed to fetch SOT URL: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { @@ -173,54 +149,50 @@ func GetTagsFromURL(url, imageTagSuffix string, nopSuffixes, ignores []string) ( } body, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + return nil, fmt.Errorf("failed to read SOT URL response body: %w", err) } var response RaneSOTResponseBody if err := json.Unmarshal(body, &response); err != nil { - return nil, fmt.Errorf("failed to parse JSON: %w", err) + return nil, fmt.Errorf("failed to parse SOT response body: %w", err) } - tags := make([]string, 0) - seenTags := make(map[string]bool, 0) + refs := make([]string, 0) + nops := make([]string, 0) + seenRefs := make(map[string]bool, 0) + seenNOPs := make(map[string]bool, 0) // check all the nodes for uniq tags for _, node := range response.Nodes { version := node.Version nop := node.NOP - // skip if we are not interested in this NOP - for _, ns := range nopSuffixes { - if !strings.Contains(nop, ns) { - continue - } - } + // add uniq NOP + if _, ok := seenNOPs[nop]; !ok { + nops = append(nops, nop) + seenNOPs[nop] = true + } + // continue if it's not the NOP we need + if nop != nopName { + continue + } // skip if version is empty if version == "" { continue } - - // skip if we ignore some images - ignored := false - for _, ignore := range ignores { - if strings.Contains(version, ignore) { - ignored = true - break - } - } - - if strings.Contains(version, imageTagSuffix) && !ignored { - if _, ok := seenTags[version]; ok { - continue - } - tags = append(tags, version) - seenTags[version] = true + // add uniq version + if _, ok := seenRefs[version]; !ok { + refs = append(refs, version) + seenRefs[version] = true } } - return tags, nil + semverTags := FilterSemverTags(refs, []string{}, exclude) + slices.Reverse(semverTags) + L.Info().Strs("NOPs", nops).Msg("Scanned NOPs") + return semverTags, nil } -// SortSemverTags parses valid versions and returns them sorted from latest to lowest -func SortSemverTags(versions []string, include []string, exclude []string) []string { +// FilterSemverTags parses valid versions and returns them sorted from latest to lowest +func FilterSemverTags(versions []string, include []string, exclude []string) []string { if len(versions) == 0 { return []string{} } @@ -234,17 +206,15 @@ func SortSemverTags(versions []string, include []string, exclude []string) []str } parsedVersions = append(parsedVersions, parsed) } - // descending order sort.Slice(parsedVersions, func(i, j int) bool { return parsedVersions[i].GreaterThan(parsedVersions[j]) }) - // fliter include/exclude filtered := filterVersions(parsedVersions, include, exclude) L.Info(). - Strs("include", include). - Strs("exclude", exclude). + Strs("Include", include). + Strs("Exclude", exclude). Msg("Applied filters") return filtered } @@ -278,4 +248,4 @@ func matchesFilter(version string, include, exclude []string) bool { } } return false -} \ No newline at end of file +} diff --git a/framework/compat_test.go b/framework/compat_test.go index b15bc0df3..95b7fea58 100644 --- a/framework/compat_test.go +++ b/framework/compat_test.go @@ -97,7 +97,7 @@ func TestSmokeTagsFilter(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - tags := SortSemverTags(tc.tags, tc.includes, tc.excludes) + tags := FilterSemverTags(tc.tags, tc.includes, tc.excludes) require.Equal(t, tc.expectedTags, tags, "Test case: %s", tc.name) }) }