diff --git a/cli/cmd/init_install_config.go b/cli/cmd/init_install_config.go index 4652af84..a7e5b884 100644 --- a/cli/cmd/init_install_config.go +++ b/cli/cmd/init_install_config.go @@ -351,7 +351,11 @@ func (c *InitInstallConfigCmd) updateConfigFromOpts(config *files.RootConfig) *f if len(c.Opts.CephHosts) > 0 { cephHosts := []files.CephHost{} for _, hostCfg := range c.Opts.CephHosts { - cephHosts = append(cephHosts, files.CephHost(hostCfg)) + cephHosts = append(cephHosts, files.CephHost{ + Hostname: hostCfg.Hostname, + IPAddress: hostCfg.IPAddress, + IsMaster: hostCfg.IsMaster, + }) } config.Ceph.Hosts = cephHosts } diff --git a/cli/cmd/update_dockerfile_test.go b/cli/cmd/update_dockerfile_test.go index 9c852223..0af6798c 100644 --- a/cli/cmd/update_dockerfile_test.go +++ b/cli/cmd/update_dockerfile_test.go @@ -139,17 +139,12 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockImageManager := system.NewMockImageManager(GinkgoT()) mockFileIO := util.NewMockFileIO(GinkgoT()) - // Create a temporary file for the Dockerfile - tempFile, err := os.CreateTemp("", "dockerfile-test-*") + pr, pw, err := os.Pipe() Expect(err).To(BeNil()) - DeferCleanup(func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }) - _, err = tempFile.WriteString(sampleDockerfileContent) + _, err = pw.WriteString(sampleDockerfileContent) Expect(err).To(BeNil()) - // Reset file position to beginning - _, _ = tempFile.Seek(0, 0) + Expect(pw.Close()).To(Succeed()) + DeferCleanup(func() { _ = pr.Close() }) c.Opts.Dockerfile = "Dockerfile" c.Opts.Baseimage = "" @@ -160,7 +155,7 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockPackageManager.EXPECT().GetBaseimagePath("", false).Return("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar", nil) mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar").Return(nil) mockPackageManager.EXPECT().FileIO().Return(mockFileIO) - mockFileIO.EXPECT().Open("Dockerfile").Return(tempFile, nil) + mockFileIO.EXPECT().Open("Dockerfile").Return(pr, nil) mockFileIO.EXPECT().WriteFile("Dockerfile", []byte("FROM ubuntu:24.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(errors.New("write failed")) err = c.UpdateDockerfile(mockPackageManager, mockImageManager, []string{}) @@ -173,17 +168,12 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockImageManager := system.NewMockImageManager(GinkgoT()) mockFileIO := util.NewMockFileIO(GinkgoT()) - // Create a temporary file for the Dockerfile - tempFile, err := os.CreateTemp("", "dockerfile-test-*") + pr, pw, err := os.Pipe() Expect(err).To(BeNil()) - DeferCleanup(func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }) - _, err = tempFile.WriteString(sampleDockerfileContent) + _, err = pw.WriteString(sampleDockerfileContent) Expect(err).To(BeNil()) - // Reset file position to beginning - _, _ = tempFile.Seek(0, 0) + Expect(pw.Close()).To(Succeed()) + DeferCleanup(func() { _ = pr.Close() }) c.Opts.Dockerfile = "Dockerfile" c.Opts.Baseimage = "" @@ -193,7 +183,7 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockPackageManager.EXPECT().GetFullImageTag("").Return("ubuntu:24.04", nil) mockPackageManager.EXPECT().GetBaseimagePath("", false).Return("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar", nil) mockPackageManager.EXPECT().FileIO().Return(mockFileIO) - mockFileIO.EXPECT().Open("Dockerfile").Return(tempFile, nil) + mockFileIO.EXPECT().Open("Dockerfile").Return(pr, nil) mockFileIO.EXPECT().WriteFile("Dockerfile", []byte("FROM ubuntu:24.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(nil) mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar").Return(nil) @@ -206,17 +196,12 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockImageManager := system.NewMockImageManager(GinkgoT()) mockFileIO := util.NewMockFileIO(GinkgoT()) - // Create a temporary file for the Dockerfile - tempFile, err := os.CreateTemp("", "dockerfile-test-*") + pr, pw, err := os.Pipe() Expect(err).To(BeNil()) - DeferCleanup(func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }) - _, err = tempFile.WriteString(sampleDockerfileContent) + _, err = pw.WriteString(sampleDockerfileContent) Expect(err).To(BeNil()) - // Reset file position to beginning - _, _ = tempFile.Seek(0, 0) + Expect(pw.Close()).To(Succeed()) + DeferCleanup(func() { _ = pr.Close() }) c.Opts.Dockerfile = "Dockerfile" c.Opts.Baseimage = "workspace-agent-20.04.tar" @@ -226,7 +211,7 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockPackageManager.EXPECT().GetFullImageTag("workspace-agent-20.04.tar").Return("ubuntu:20.04", nil) mockPackageManager.EXPECT().GetBaseimagePath("workspace-agent-20.04.tar", true).Return("/test/workdir/deps/codesphere/images/workspace-agent-20.04.tar", nil) mockPackageManager.EXPECT().FileIO().Return(mockFileIO) - mockFileIO.EXPECT().Open("Dockerfile").Return(tempFile, nil) + mockFileIO.EXPECT().Open("Dockerfile").Return(pr, nil) mockFileIO.EXPECT().WriteFile("Dockerfile", []byte("FROM ubuntu:20.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(nil) mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-20.04.tar").Return(nil) @@ -239,17 +224,12 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockImageManager := system.NewMockImageManager(GinkgoT()) mockFileIO := util.NewMockFileIO(GinkgoT()) - // Create a temporary file for the Dockerfile - tempFile, err := os.CreateTemp("", "dockerfile-test-*") + pr, pw, err := os.Pipe() Expect(err).To(BeNil()) - DeferCleanup(func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }) - _, err = tempFile.WriteString(sampleDockerfileContent) + _, err = pw.WriteString(sampleDockerfileContent) Expect(err).To(BeNil()) - // Reset file position to beginning - _, _ = tempFile.Seek(0, 0) + Expect(pw.Close()).To(Succeed()) + DeferCleanup(func() { _ = pr.Close() }) c.Opts.Dockerfile = "custom/Dockerfile" c.Opts.Baseimage = "workspace-agent-24.04.tar" @@ -259,7 +239,7 @@ var _ = Describe("UpdateDockerfileCmd", func() { mockPackageManager.EXPECT().GetFullImageTag("workspace-agent-24.04.tar").Return("registry.example.com/workspace-agent:24.04", nil) mockPackageManager.EXPECT().GetBaseimagePath("workspace-agent-24.04.tar", false).Return("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar", nil) mockPackageManager.EXPECT().FileIO().Return(mockFileIO) - mockFileIO.EXPECT().Open("custom/Dockerfile").Return(tempFile, nil) + mockFileIO.EXPECT().Open("custom/Dockerfile").Return(pr, nil) mockFileIO.EXPECT().WriteFile("custom/Dockerfile", []byte("FROM registry.example.com/workspace-agent:24.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(nil) mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar").Return(nil) diff --git a/docs/oms_beta_bootstrap-gcp.md b/docs/oms_beta_bootstrap-gcp.md index 7dba461b..797ed0bb 100644 --- a/docs/oms_beta_bootstrap-gcp.md +++ b/docs/oms_beta_bootstrap-gcp.md @@ -31,7 +31,7 @@ oms beta bootstrap-gcp [flags] --datacenter-name string Datacenter name (default: dev) (default "dev") --dns-project-id string GCP Project ID for Cloud DNS (optional) --dns-zone-name string Cloud DNS Zone Name (optional) (default "oms-testing") - --experiments stringArray Experiments to enable in Codesphere installation (optional) (default [managed-services,headless-services,vcluster,custom-service-image,ms-in-ls,secret-management,sub-path-mount]) + --experiments stringArray Experiments to enable in Codesphere installation (optional) (default [managed-services,custom-service-image,ms-in-ls,secret-management,sub-path-mount]) --external-loki-endpoint string External Loki endpoint for Grafana Alloy log forwarding (optional) --external-loki-secret string External Loki password stored in the generated vault (optional) --external-loki-user string External Loki username for Grafana Alloy log forwarding (optional) diff --git a/docs/oms_beta_bootstrap-local.md b/docs/oms_beta_bootstrap-local.md index 9703d33a..df6c25a8 100644 --- a/docs/oms_beta_bootstrap-local.md +++ b/docs/oms_beta_bootstrap-local.md @@ -17,7 +17,7 @@ oms beta bootstrap-local [flags] ``` --base-domain string Base domain for Codesphere (default "cs.local") - --experiments stringArray Experiments to enable in Codesphere installation (optional) (default [managed-services,headless-services,vcluster,custom-service-image,ms-in-ls,secret-management,sub-path-mount]) + --experiments stringArray Experiments to enable in Codesphere installation (optional) (default [managed-services,custom-service-image,ms-in-ls,secret-management,sub-path-mount]) --feature-flags stringArray Feature flags to enable in Codesphere installation (optional) -h, --help help for bootstrap-local --install-config string Path to install config file (default: /config.yaml) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 81fe1658..d8af7920 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -62,8 +62,6 @@ func GetDNSRecordNames(baseDomain string) []struct { var DefaultExperiments []string = []string{ "managed-services", - "headless-services", - "vcluster", "custom-service-image", "ms-in-ls", "secret-management", @@ -80,9 +78,10 @@ type GCPBootstrapper struct { // Environment Env *CodesphereEnvironment // SSH command runner - NodeClient node.NodeClient - PortalClient portal.Portal - GitHubClient github.GitHubClient + NodeClient node.NodeClient + PortalClient portal.Portal + GitHubClient github.GitHubClient + OmsBinaryBuilder func() (string, func(), error) } type CodesphereEnvironment struct { @@ -182,16 +181,17 @@ func NewGCPBootstrapper( gitHubClient github.GitHubClient, ) (*GCPBootstrapper, error) { return &GCPBootstrapper{ - ctx: ctx, - stlog: stlog, - fw: fw, - icg: icg, - GCPClient: gcpClient, - Env: CodesphereEnv, - NodeClient: sshRunner, - PortalClient: portalClient, - Time: time, - GitHubClient: gitHubClient, + ctx: ctx, + stlog: stlog, + fw: fw, + icg: icg, + GCPClient: gcpClient, + Env: CodesphereEnv, + NodeClient: sshRunner, + PortalClient: portalClient, + Time: time, + GitHubClient: gitHubClient, + OmsBinaryBuilder: BuildOmsLinuxBinary, }, nil } @@ -922,6 +922,16 @@ func (b *GCPBootstrapper) InstallCodesphere() error { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } + if ltsSpec := FindLTSSpec(b.Env.InstallVersion); ltsSpec != nil { + if ltsSpec.RequiresOmsBinaryUpdate { + if err := b.ensureNewOmsBinaryOnJumpbox(); err != nil { + return fmt.Errorf("failed to update OMS binary on jumpbox for %s: %w", b.Env.InstallVersion, err) + } + } + b.startLTSCephMasterWatcher() + defer b.stopLTSCephMasterWatcher() + } + err = b.runInstallCommand(fullPackageFilename) if err != nil { return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) @@ -930,6 +940,61 @@ func (b *GCPBootstrapper) InstallCodesphere() error { return nil } +// ensureNewOmsBinaryOnJumpbox copies a freshly-built linux/amd64 OMS binary to +// the jumpbox, replacing the old installed version. +func (b *GCPBootstrapper) ensureNewOmsBinaryOnJumpbox() error { + b.stlog.Logf("Updating OMS binary on jumpbox for %s compatibility...", b.Env.InstallVersion) + + binaryPath, cleanup, err := b.OmsBinaryBuilder() + if err != nil { + return fmt.Errorf("failed to prepare OMS linux binary: %w", err) + } + defer cleanup() + + const remoteTmpPath = "/tmp/oms-new" + if err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, binaryPath, remoteTmpPath); err != nil { + return fmt.Errorf("failed to copy OMS binary to jumpbox: %w", err) + } + + if err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("chmod +x %s && mv %s /usr/local/bin/oms", remoteTmpPath, remoteTmpPath)); err != nil { + return fmt.Errorf("failed to install OMS binary on jumpbox: %w", err) + } + + return nil +} + +// startLTSCephMasterWatcher starts a background process on the ceph master node that continuously +// re-adds the master to the Ceph orchestrator host inventory. This is required for LTS versions +// because the installer's configureHosts step applies a declarative host spec containing only the +// non-master nodes, which removes the master from the inventory. The watcher restores it within +// seconds, before the subsequent configureMonitors step runs. +func (b *GCPBootstrapper) startLTSCephMasterWatcher() { + if len(b.Env.CephNodes) == 0 || len(b.Env.InstallConfig.Ceph.Hosts) == 0 { + return + } + masterHost := b.Env.InstallConfig.Ceph.Hosts[0] + // Use cephadm shell (same as the installer) so the command runs inside the ceph container, + // bypassing any standalone-binary or keyring availability issues on the host. + // The FSID is auto-detected from /var/lib/ceph/; all output is logged for diagnostics. + cmd := fmt.Sprintf( + `nohup bash -c "while true; do FSID=\$(ls /var/lib/ceph/ 2>/dev/null | head -1); [ -n \"\$FSID\" ] && [ -x /usr/local/bin/cephadm ] && /usr/local/bin/cephadm shell --fsid \"\$FSID\" -- ceph orch host add %s %s 2>&1; sleep 3; done" > /tmp/ceph-host-watcher.log 2>&1 & echo $! > /tmp/ceph-host-watcher.pid`, + masterHost.Hostname, + masterHost.IPAddress, + ) + if err := b.Env.CephNodes[0].RunSSHCommand("root", cmd); err != nil { + b.stlog.Logf("Note: could not start ceph master host watcher on %s: %v", masterHost.Hostname, err) + } +} + +// stopLTSCephMasterWatcher stops the background watcher started by startLTSCephMasterWatcher. +func (b *GCPBootstrapper) stopLTSCephMasterWatcher() { + if len(b.Env.CephNodes) == 0 || len(b.Env.InstallConfig.Ceph.Hosts) == 0 { + return + } + cmd := `kill $(cat /tmp/ceph-host-watcher.pid 2>/dev/null) 2>/dev/null; rm -f /tmp/ceph-host-watcher.pid /tmp/ceph-host-watcher.log` + _ = b.Env.CephNodes[0].RunSSHCommand("root", cmd) +} + func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { packageFilename := "installer.tar.gz" if b.Env.RegistryType == RegistryTypeGitHub { @@ -959,7 +1024,13 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { packageFilename, b.Env.InstallHash, b.Env.InstallVersion) err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd) if err != nil { - return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err) + // A stale partial file from a previous (different) build can cause MD5 verification to + // fail even after a successful byte-range resume. Delete it and retry from scratch. + b.stlog.Logf("Download failed; removing any stale partial file and retrying from scratch...") + cleanAndRetryCmd := fmt.Sprintf("rm -f %s && %s", fullPackageFilename, downloadCmd) + if retryErr := b.Env.Jumpbox.RunSSHCommand("root", cleanAndRetryCmd); retryErr != nil { + return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", retryErr) + } } return fullPackageFilename, nil diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index 6247faf3..0fa073d9 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -263,7 +263,7 @@ var _ = Describe("GCP Bootstrapper", func() { Expect(cpNode.GetInternalIP()).To(Equal("10.0.0.1")) } - Expect(len(bs.Env.InstallConfig.Codesphere.ManagedServices)).To(Equal(4)) + Expect(len(bs.Env.InstallConfig.Codesphere.ManagedServices)).To(Equal(5)) }) }) @@ -1227,6 +1227,55 @@ var _ = Describe("GCP Bootstrapper", func() { Expect(err).NotTo(HaveOccurred()) }) + Context("LTS 1.77.2", func() { + BeforeEach(func() { + csEnv.InstallVersion = "codesphere-lts-v1.77.2" + }) + JustBeforeEach(func() { + // Inject a stub binary builder so tests don't invoke `go build`. + bs.OmsBinaryBuilder = func() (string, func(), error) { + f, err := os.CreateTemp("", "oms-test-binary-*") + Expect(err).NotTo(HaveOccurred()) + Expect(f.Close()).To(Succeed()) + return f.Name(), func() { Expect(os.Remove(f.Name())).To(Succeed()) }, nil + } + }) + It("downloads package, updates OMS binary, and installs codesphere", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms download package -f installer.tar.gz -H abc1234567890 codesphere-lts-v1.77.2").Return(nil) + nodeClient.EXPECT().CopyFile(mock.MatchedBy(jumpboxMatcher), mock.Anything, "/tmp/oms-new").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "chmod +x /tmp/oms-new && mv /tmp/oms-new /usr/local/bin/oms").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p codesphere-lts-v1.77.2-abc1234567890-installer.tar.gz").Return(nil) + + err := bs.InstallCodesphere() + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails when OmsBinaryBuilder fails", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms download package -f installer.tar.gz -H abc1234567890 codesphere-lts-v1.77.2").Return(nil) + bs.OmsBinaryBuilder = func() (string, func(), error) { + return "", func() {}, fmt.Errorf("build failed") + } + + err := bs.InstallCodesphere() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to update OMS binary on jumpbox for codesphere-lts-v1.77.2")) + }) + + It("fails when copying binary to jumpbox fails", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms download package -f installer.tar.gz -H abc1234567890 codesphere-lts-v1.77.2").Return(nil) + nodeClient.EXPECT().CopyFile(mock.MatchedBy(jumpboxMatcher), mock.Anything, "/tmp/oms-new").Return(fmt.Errorf("copy failed")) + + err := bs.InstallCodesphere() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to update OMS binary on jumpbox for codesphere-lts-v1.77.2")) + }) + }) + Context("with local package", func() { BeforeEach(func() { csEnv.InstallLocal = "fake-installer-lite.tar.gz" @@ -1288,8 +1337,23 @@ var _ = Describe("GCP Bootstrapper", func() { }) }) + It("retries with a clean slate when download fails, succeeds on retry", func() { + downloadCmd := "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3" + cleanAndRetryCmd := "rm -f v1.2.3-abc1234567890-installer.tar.gz && " + downloadCmd + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", downloadCmd).Return(fmt.Errorf("md5 mismatch")).Once() + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", cleanAndRetryCmd).Return(nil).Once() + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-abc1234567890-installer.tar.gz").Return(nil).Once() + + err := bs.InstallCodesphere() + Expect(err).NotTo(HaveOccurred()) + }) + It("fails when download package fails", func() { - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(fmt.Errorf("download error")) + downloadCmd := "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3" + cleanAndRetryCmd := "rm -f v1.2.3-abc1234567890-installer.tar.gz && " + downloadCmd + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", downloadCmd).Return(fmt.Errorf("download error")).Once() + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", cleanAndRetryCmd).Return(fmt.Errorf("download error on retry")).Once() err := bs.InstallCodesphere() Expect(err).To(HaveOccurred()) diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 542bf404..8f4b54fb 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -130,14 +130,17 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { Hostname: b.Env.CephNodes[0].GetName(), IsMaster: true, IPAddress: b.Env.CephNodes[0].GetInternalIP(), + SSHPort: 22, }, { Hostname: b.Env.CephNodes[1].GetName(), IPAddress: b.Env.CephNodes[1].GetInternalIP(), + SSHPort: 22, }, { Hostname: b.Env.CephNodes[2].GetName(), IPAddress: b.Env.CephNodes[2].GetInternalIP(), + SSHPort: 22, }, } b.Env.InstallConfig.Ceph.OSDs = []files.CephOSD{ @@ -163,18 +166,21 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { ControlPlanes: []files.K8sNode{ { IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + SSHPort: 22, }, }, Workers: []files.K8sNode{ { IPAddress: b.Env.ControlPlaneNodes[0].GetInternalIP(), + SSHPort: 22, }, - { IPAddress: b.Env.ControlPlaneNodes[1].GetInternalIP(), + SSHPort: 22, }, { IPAddress: b.Env.ControlPlaneNodes[2].GetInternalIP(), + SSHPort: 22, }, }, } @@ -333,6 +339,8 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { b.Env.InstallConfig.Codesphere.Features = b.Env.FeatureFlags b.applyExternalLokiConfig() + b.Env.InstallConfig.GeneratedForVersion = b.Env.InstallVersion + if !b.Env.ExistingConfigUsed { err := b.icg.GenerateSecrets() if err != nil { @@ -368,16 +376,25 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { return fmt.Errorf("failed to write config file: %w", err) } + jumpboxConfigLocalPath := b.Env.InstallConfigPath + if ltsSpec := FindLTSSpec(b.Env.InstallVersion); ltsSpec != nil && ltsSpec.RequiresJumpboxFiles { + // Old LTS installers: inline compat-stripped codesphere in jumpbox config. + var err error + jumpboxConfigLocalPath, err = b.writeLTSJumpboxFiles(ltsSpec) + if err != nil { + return err + } + } + if err := b.icg.WriteVault(b.Env.SecretsFilePath, true); err != nil { return fmt.Errorf("failed to write vault file: %w", err) } - err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallConfigPath, remoteInstallConfigPath) - if err != nil { + if err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, jumpboxConfigLocalPath, remoteInstallConfigPath); err != nil { return fmt.Errorf("failed to copy install config to jumpbox: %w", err) } - err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml") + err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml") if err != nil { return fmt.Errorf("failed to copy secrets file to jumpbox: %w", err) } @@ -493,6 +510,22 @@ func (b *GCPBootstrapper) EncryptVault() error { return nil } +// writeLTSJumpboxFiles generates the LTS-versioned config-lts-.yaml locally +// and returns its path for copying to the jumpbox. +func (b *GCPBootstrapper) writeLTSJumpboxFiles(spec *LTSSpec) (jumpboxConfigLocalPath string, err error) { + jumpboxConfigBytes, err := GenerateLTSJumpboxFiles(b.Env.InstallConfig, spec) + if err != nil { + return "", fmt.Errorf("failed to prepare %s jumpbox config: %w", spec.InstallVersion, err) + } + + jumpboxConfigLocalPath = LocalLTSConfigPath(b.Env.InstallConfigPath, spec) + if err := b.fw.CreateAndWrite(jumpboxConfigLocalPath, jumpboxConfigBytes, "Jumpbox Config ("+spec.InstallVersion+")"); err != nil { + return "", fmt.Errorf("failed to write %s jumpbox config file: %w", spec.InstallVersion, err) + } + + return jumpboxConfigLocalPath, nil +} + // decryptVault creates an unencrypted copy of the vault in dst on the jumpbox // Make sure to delete the unencrypted file when not needed anymore. func (b *GCPBootstrapper) decryptVault(dst string) error { diff --git a/internal/bootstrap/gcp/install_config_test.go b/internal/bootstrap/gcp/install_config_test.go index 4e661f13..4dcced98 100644 --- a/internal/bootstrap/gcp/install_config_test.go +++ b/internal/bootstrap/gcp/install_config_test.go @@ -726,6 +726,76 @@ var _ = Describe("Installconfig & Secrets", func() { Expect(bs.Env.InstallConfig.Codesphere.OpenBao.Engine).To(Equal("fake-engine")) }) }) + + Context("When install version is codesphere-lts-v1.77.2", func() { + BeforeEach(func() { + csEnv.InstallVersion = "codesphere-lts-v1.77.2" + csEnv.Experiments = gcp.DefaultExperiments + csEnv.InstallConfig.Codesphere.ManagedServices = []files.ManagedServiceConfig{ + { + Name: "postgres", + Version: "v1", + Author: "Codesphere", + DisplayName: "PostgreSQL", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{Endpoint: "http://ms-backend:3000"}, + }, + }, + { + Name: "s3", + Version: "v2", + }, + } + }) + It("generates the LTS jumpbox files and copies them to the jumpbox", func() { + icg.EXPECT().GenerateSecrets().Return(nil) + icg.EXPECT().WriteInstallConfig("fake-config-file", true).Return(nil) + fw.EXPECT().CreateAndWrite("config-lts-1_77_2.yaml", mock.Anything, mock.Anything).Return(nil) + icg.EXPECT().WriteVault("fake-secret", true).Return(nil) + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil).Twice() + + err := bs.UpdateInstallConfig() + Expect(err).NotTo(HaveOccurred()) + }) + It("does not modify the in-memory codesphere config for LTS 1.77.2", func() { + icg.EXPECT().GenerateSecrets().Return(nil) + icg.EXPECT().WriteInstallConfig("fake-config-file", true).Return(nil) + fw.EXPECT().CreateAndWrite(mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + icg.EXPECT().WriteVault("fake-secret", true).Return(nil) + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil).Twice() + + err := bs.UpdateInstallConfig() + Expect(err).NotTo(HaveOccurred()) + + // In-memory config must be preserved (compat is applied only to jumpbox files) + Expect(bs.Env.InstallConfig.Codesphere.Experiments).To(ContainElement("secret-management")) + Expect(bs.Env.InstallConfig.Codesphere.Experiments).To(ContainElement("sub-path-mount")) + Expect(bs.Env.InstallConfig.CodesphereConfigPath).To(BeEmpty()) + + services := bs.Env.InstallConfig.Codesphere.ManagedServices + Expect(services[0].Author).To(Equal("Codesphere")) + Expect(services[0].DisplayName).To(Equal("PostgreSQL")) + Expect(services[0].Backend.API.Endpoint).To(Equal("http://ms-backend:3000")) + }) + }) + + Context("When install version is not codesphere-lts-v1.77.2", func() { + BeforeEach(func() { + csEnv.InstallVersion = "master" + csEnv.Experiments = gcp.DefaultExperiments + }) + It("uses the regular config.yaml directly (inline codesphere object)", func() { + icg.EXPECT().GenerateSecrets().Return(nil) + icg.EXPECT().WriteInstallConfig("fake-config-file", true).Return(nil) + icg.EXPECT().WriteVault("fake-secret", true).Return(nil) + // config.yaml → /etc/codesphere/config.yaml, prod.vault.yaml → secrets dir + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil).Twice() + + err := bs.UpdateInstallConfig() + Expect(err).NotTo(HaveOccurred()) + }) + }) + Context("When external Loki config is set", func() { BeforeEach(func() { csEnv.ExternalLokiEndpoint = "https://loki.example.com/loki/api/v1/push" diff --git a/internal/bootstrap/gcp/lts.go b/internal/bootstrap/gcp/lts.go new file mode 100644 index 00000000..d1914f78 --- /dev/null +++ b/internal/bootstrap/gcp/lts.go @@ -0,0 +1,121 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +// LTSSpec describes the compatibility requirements for a specific LTS release. +// To add support for a new LTS version, add a new entry to ltsRegistry — no other +// files need to change. +type LTSSpec struct { + // InstallVersion is the exact install version string that identifies this LTS release. + InstallVersion string + // UnsupportedExperiments lists experiments that did not exist at this LTS release + // and must be stripped from the config before passing it to the LTS installer. + UnsupportedExperiments []string + // ClearManagedServices instructs the compat layer to set ManagedServices to nil. + // Required when the LTS schema expects full provider definitions, not the abbreviated + // form stored in config.yaml. + ClearManagedServices bool + // RequiresJumpboxFiles instructs the bootstrap to generate LTS-versioned compat config files + // (e.g. config-lts-1_77_2.yaml and codesphere-lts-1_77_2.yaml) instead of using config.yaml directly. + RequiresJumpboxFiles bool + // RequiresOmsBinaryUpdate instructs the bootstrap to build a fresh linux/amd64 OMS binary + // and deploy it to the jumpbox before running the installer. + RequiresOmsBinaryUpdate bool +} + +// ltsRegistry is the single source of truth for all known LTS versions and their quirks. +// Add a new LTSSpec entry here to support an additional LTS release. +var ltsRegistry = []LTSSpec{ + { + InstallVersion: "codesphere-lts-v1.77.2", + UnsupportedExperiments: []string{ + "secret-management", + "sub-path-mount", + }, + ClearManagedServices: true, + RequiresJumpboxFiles: true, + RequiresOmsBinaryUpdate: true, + }, +} + +// FindLTSSpec returns the LTSSpec for the given installVersion, or nil if it is not a +// known LTS release that requires special handling. +func FindLTSSpec(installVersion string) *LTSSpec { + for i := range ltsRegistry { + if ltsRegistry[i].InstallVersion == installVersion { + return <sRegistry[i] + } + } + return nil +} + +// LTSConfigFileSuffix derives a filesystem-safe suffix from an LTS installVersion string. +// For example, "codesphere-lts-v1.77.2" -> "lts-1_77_2". +func LTSConfigFileSuffix(installVersion string) string { + s := strings.TrimPrefix(installVersion, "codesphere-") + s = strings.ReplaceAll(s, "v", "") + s = strings.ReplaceAll(s, ".", "_") + return s +} + +// LocalLTSConfigPath derives the local path for the LTS-specific jumpbox config from the +// main config path. For example, with installVersion "codesphere-lts-v1.77.2" and +// configPath "config.yaml" it returns "config-lts-1_77_2.yaml". +func LocalLTSConfigPath(configPath string, spec *LTSSpec) string { + return filepath.Join(filepath.Dir(configPath), "config-"+LTSConfigFileSuffix(spec.InstallVersion)+".yaml") +} + +// GenerateLTSJumpboxFiles generates the LTS-versioned config file needed on the jumpbox +// without modifying the original root config. It returns the bytes for +// config-lts-.yaml with the compat-stripped codesphere section inlined. +func GenerateLTSJumpboxFiles(root *files.RootConfig, spec *LTSSpec) (jumpboxConfigBytes []byte, err error) { + csCopy := root.Codesphere + ApplyLTSCompat(&csCopy, spec) + + rootCopy := *root + rootCopy.Codesphere = csCopy + // Clear the version annotation so the old LTS installer does not encounter an unknown field. + rootCopy.GeneratedForVersion = "" + + jumpboxConfigBytes, err = rootCopy.Marshal() + if err != nil { + return nil, fmt.Errorf("failed to marshal %s jumpbox config: %w", spec.InstallVersion, err) + } + + return jumpboxConfigBytes, nil +} + +// ApplyLTSCompat adjusts a CodesphereConfig to be compatible with the given LTS release: +// 1. Experiments not present in the LTS release are removed. +// 2. ManagedServices is cleared when the LTS schema requires full provider definitions. +func ApplyLTSCompat(cfg *files.CodesphereConfig, spec *LTSSpec) { + cfg.Experiments = FilterExperiments(cfg.Experiments, spec.UnsupportedExperiments) + if spec.ClearManagedServices { + cfg.ManagedServices = nil + } +} + +// FilterExperiments returns a new slice of experiments with the unsupported ones removed. +func FilterExperiments(experiments, unsupported []string) []string { + unsupportedSet := make(map[string]struct{}, len(unsupported)) + for _, u := range unsupported { + unsupportedSet[u] = struct{}{} + } + + filtered := make([]string, 0, len(experiments)) + for _, exp := range experiments { + if _, remove := unsupportedSet[exp]; !remove { + filtered = append(filtered, exp) + } + } + return filtered +} diff --git a/internal/bootstrap/gcp/lts_test.go b/internal/bootstrap/gcp/lts_test.go new file mode 100644 index 00000000..04f4c6f6 --- /dev/null +++ b/internal/bootstrap/gcp/lts_test.go @@ -0,0 +1,221 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/installer/files" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("LTS Compatibility", func() { + Describe("FindLTSSpec", func() { + It("returns a spec for the LTS 1.77.2 version string", func() { + spec := gcp.FindLTSSpec("codesphere-lts-v1.77.2") + Expect(spec).NotTo(BeNil()) + Expect(spec.InstallVersion).To(Equal("codesphere-lts-v1.77.2")) + Expect(spec.RequiresJumpboxFiles).To(BeTrue()) + Expect(spec.RequiresOmsBinaryUpdate).To(BeTrue()) + Expect(spec.ClearManagedServices).To(BeTrue()) + }) + + It("returns nil for another LTS version", func() { + Expect(gcp.FindLTSSpec("codesphere-lts-v1.80.0")).To(BeNil()) + }) + + It("returns nil for an empty string", func() { + Expect(gcp.FindLTSSpec("")).To(BeNil()) + }) + + It("returns nil for a non-LTS version", func() { + Expect(gcp.FindLTSSpec("master")).To(BeNil()) + }) + + It("returns nil for a partial match", func() { + Expect(gcp.FindLTSSpec("codesphere-lts-v1.77.2-extra")).To(BeNil()) + }) + }) + + Describe("FilterExperiments", func() { + It("removes unsupported experiments", func() { + input := []string{"managed-services", "custom-service-image", "secret-management", "ms-in-ls", "sub-path-mount"} + unsupported := []string{"secret-management", "sub-path-mount"} + result := gcp.FilterExperiments(input, unsupported) + Expect(result).To(ConsistOf("managed-services", "custom-service-image", "ms-in-ls")) + }) + + It("returns the same slice when nothing is unsupported", func() { + input := []string{"managed-services", "custom-service-image"} + result := gcp.FilterExperiments(input, []string{}) + Expect(result).To(ConsistOf("managed-services", "custom-service-image")) + }) + + It("returns empty slice when all experiments are unsupported", func() { + input := []string{"secret-management", "sub-path-mount"} + unsupported := []string{"secret-management", "sub-path-mount"} + result := gcp.FilterExperiments(input, unsupported) + Expect(result).To(BeEmpty()) + }) + + It("handles empty input", func() { + result := gcp.FilterExperiments([]string{}, []string{"secret-management"}) + Expect(result).To(BeEmpty()) + }) + }) + + Describe("ApplyLTSCompat", func() { + var spec *gcp.LTSSpec + + BeforeEach(func() { + spec = gcp.FindLTSSpec("codesphere-lts-v1.77.2") + Expect(spec).NotTo(BeNil()) + }) + + It("removes unsupported experiments from the config", func() { + cfg := &files.CodesphereConfig{ + Experiments: []string{"managed-services", "custom-service-image", "secret-management", "ms-in-ls", "sub-path-mount"}, + } + gcp.ApplyLTSCompat(cfg, spec) + Expect(cfg.Experiments).To(ConsistOf("managed-services", "custom-service-image", "ms-in-ls")) + Expect(cfg.Experiments).NotTo(ContainElement("secret-management")) + Expect(cfg.Experiments).NotTo(ContainElement("sub-path-mount")) + }) + + It("clears managed services (LTS 1.77.2 schema requires full provider definitions)", func() { + cfg := &files.CodesphereConfig{ + ManagedServices: []files.ManagedServiceConfig{ + { + Name: "postgres", + Version: "v1", + Author: "Codesphere", + DisplayName: "PostgreSQL", + Description: "Open-source database", + Category: "Database", + Scope: "global", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{ + Endpoint: "http://ms-backend-postgres.postgres-operator:3000/api/v1/postgres", + }, + }, + Plans: []files.ServicePlan{{ID: 0, Name: "Small"}}, + Capabilities: &files.ManagedServiceCapabilities{ + Pause: true, + Backups: true, + }, + }, + { + Name: "s3", + Version: "v1", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{ + Endpoint: "http://localhost:8080", + }, + }, + }, + }, + } + + gcp.ApplyLTSCompat(cfg, spec) + + Expect(cfg.ManagedServices).To(BeNil()) + }) + + It("handles nil managed services slice", func() { + cfg := &files.CodesphereConfig{ + ManagedServices: nil, + Experiments: []string{"custom-service-image"}, + } + Expect(func() { gcp.ApplyLTSCompat(cfg, spec) }).NotTo(Panic()) + Expect(cfg.ManagedServices).To(BeEmpty()) + }) + + It("handles empty experiments slice", func() { + cfg := &files.CodesphereConfig{ + Experiments: []string{}, + } + gcp.ApplyLTSCompat(cfg, spec) + Expect(cfg.Experiments).To(BeEmpty()) + }) + }) + + Describe("GenerateLTSJumpboxFiles", func() { + var ( + root *files.RootConfig + spec *gcp.LTSSpec + ) + + BeforeEach(func() { + spec = gcp.FindLTSSpec("codesphere-lts-v1.77.2") + Expect(spec).NotTo(BeNil()) + + root = &files.RootConfig{ + Codesphere: files.CodesphereConfig{ + Experiments: []string{"managed-services", "custom-service-image", "secret-management", "ms-in-ls", "sub-path-mount"}, + ManagedServices: []files.ManagedServiceConfig{ + {Name: "postgres", Version: "v1", Author: "Codesphere"}, + {Name: "s3", Version: "v1"}, + }, + }, + } + }) + + It("does not modify the original root config", func() { + _, err := gcp.GenerateLTSJumpboxFiles(root, spec) + Expect(err).NotTo(HaveOccurred()) + Expect(root.Codesphere.Experiments).To(ContainElement("secret-management")) + Expect(root.Codesphere.ManagedServices[0].Author).To(Equal("Codesphere")) + Expect(root.CodesphereConfigPath).To(BeEmpty()) + }) + + It("clears GeneratedForVersion in the LTS config so the old installer ignores it", func() { + root.GeneratedForVersion = "codesphere-lts-v1.77.2" + jumpboxBytes, err := gcp.GenerateLTSJumpboxFiles(root, spec) + Expect(err).NotTo(HaveOccurred()) + Expect(string(jumpboxBytes)).NotTo(ContainSubstring("generatedForVersion")) + // Original must not be modified + Expect(root.GeneratedForVersion).To(Equal("codesphere-lts-v1.77.2")) + }) + + It("returns codesphere bytes with compat applied", func() { + jumpboxBytes, err := gcp.GenerateLTSJumpboxFiles(root, spec) + Expect(err).NotTo(HaveOccurred()) + Expect(jumpboxBytes).NotTo(BeEmpty()) + Expect(string(jumpboxBytes)).NotTo(ContainSubstring("managedServices")) + Expect(string(jumpboxBytes)).NotTo(ContainSubstring("secret-management")) + }) + + It("returns jumpbox config bytes with inline compat-stripped codesphere", func() { + jumpboxBytes, err := gcp.GenerateLTSJumpboxFiles(root, spec) + Expect(err).NotTo(HaveOccurred()) + Expect(jumpboxBytes).NotTo(BeEmpty()) + Expect(string(jumpboxBytes)).NotTo(ContainSubstring("/etc/codesphere/codesphere.yaml")) + Expect(string(jumpboxBytes)).NotTo(ContainSubstring("managedServices")) + }) + + It("jumpbox config bytes do not contain unsupported experiment", func() { + jumpboxBytes, err := gcp.GenerateLTSJumpboxFiles(root, spec) + Expect(err).NotTo(HaveOccurred()) + Expect(string(jumpboxBytes)).NotTo(ContainSubstring("secret-management")) + }) + }) + + Describe("LTSConfigFileSuffix", func() { + It("converts the LTS 1.77.2 version string to a filesystem-safe suffix", func() { + Expect(gcp.LTSConfigFileSuffix("codesphere-lts-v1.77.2")).To(Equal("lts-1_77_2")) + }) + }) + + Describe("LocalLTSConfigPath", func() { + It("returns config-lts-1_77_2.yaml in same directory as config.yaml", func() { + spec := gcp.FindLTSSpec("codesphere-lts-v1.77.2") + Expect(gcp.LocalLTSConfigPath("config.yaml", spec)).To(Equal("config-lts-1_77_2.yaml")) + }) + + It("uses the directory of the given config path", func() { + spec := gcp.FindLTSSpec("codesphere-lts-v1.77.2") + Expect(gcp.LocalLTSConfigPath("/etc/codesphere/config.yaml", spec)).To(Equal("/etc/codesphere/config-lts-1_77_2.yaml")) + }) + }) +}) diff --git a/internal/bootstrap/gcp/oms_binary.go b/internal/bootstrap/gcp/oms_binary.go new file mode 100644 index 00000000..af884b3d --- /dev/null +++ b/internal/bootstrap/gcp/oms_binary.go @@ -0,0 +1,50 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp + +import ( + "fmt" + "os" + "os/exec" + "runtime" +) + +// BuildOmsLinuxBinary returns the path to an OMS binary built for linux/amd64. +func BuildOmsLinuxBinary() (path string, cleanup func(), err error) { + noop := func() {} + + if runtime.GOOS == "linux" && runtime.GOARCH == "amd64" { + execPath, err := os.Executable() + if err != nil { + return "", noop, fmt.Errorf("failed to locate current OMS binary: %w", err) + } + return execPath, noop, nil + } + + // Cross-compile for linux/amd64 from the current working directory (project root). + cwd, err := os.Getwd() + if err != nil { + return "", noop, fmt.Errorf("failed to determine project directory: %w", err) + } + + outFile, err := os.CreateTemp("", "oms-linux-amd64-*") + if err != nil { + return "", noop, fmt.Errorf("failed to create temp file for OMS binary: %w", err) + } + if err = outFile.Close(); err != nil { + return "", noop, fmt.Errorf("failed to close temp file for OMS binary: %w", err) + } + outPath := outFile.Name() + rmCleanup := func() { _ = os.Remove(outPath) } + + cmd := exec.Command("go", "build", "-o", outPath, "./cli") + cmd.Dir = cwd + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64") + if output, cmdErr := cmd.CombinedOutput(); cmdErr != nil { + rmCleanup() + return "", noop, fmt.Errorf("failed to cross-compile OMS binary for linux/amd64: %w\n%s", cmdErr, output) + } + + return outPath, rmCleanup, nil +} diff --git a/internal/bootstrap/local/installer.go b/internal/bootstrap/local/installer.go index e087eeb6..77ae6031 100644 --- a/internal/bootstrap/local/installer.go +++ b/internal/bootstrap/local/installer.go @@ -155,7 +155,7 @@ func (b *LocalBootstrapper) PrepareInstallerBundle() (string, error) { return destDir, nil } - log.Printf("Extracting installer bundle %s → %s", bundlePath, destDir) + log.Printf("Extracting installer bundle %s -> %s", bundlePath, destDir) if err := util.ExtractTarGz(b.fw, bundlePath, destDir); err != nil { return "", fmt.Errorf("failed to extract installer bundle: %w", err) } @@ -235,10 +235,10 @@ func symlinkBinary(name, target string) error { } if err := os.Symlink(localPath, target); err != nil { - return fmt.Errorf("failed to symlink %q → %q: %w", target, localPath, err) + return fmt.Errorf("failed to symlink %q -> %q: %w", target, localPath, err) } - log.Printf("Symlinked %s → %s", target, localPath) + log.Printf("Symlinked %s -> %s", target, localPath) return nil } @@ -422,7 +422,7 @@ func (b *LocalBootstrapper) RunInstaller() (err error) { if b.fw.Exists(depsDir) { log.Printf("deps directory already exists at %s, skipping extraction", depsDir) } else { - log.Printf("Extracting deps.tar.gz → %s", depsDir) + log.Printf("Extracting deps.tar.gz -> %s", depsDir) if err := util.ExtractTarGz(b.fw, archivePath, depsDir); err != nil { return fmt.Errorf("failed to extract deps.tar.gz: %w", err) } diff --git a/internal/installer/config_manager.go b/internal/installer/config_manager.go index 017824da..63231e6b 100644 --- a/internal/installer/config_manager.go +++ b/internal/installer/config_manager.go @@ -384,9 +384,21 @@ func (g *InstallConfig) MergeVaultIntoConfig() error { // GitHub secrets if secret, ok := secretsMap["githubAppsClientId"]; ok && secret.Fields != nil { + if g.Config.Codesphere.GitProviders == nil { + g.Config.Codesphere.GitProviders = &files.GitProvidersConfig{} + } + if g.Config.Codesphere.GitProviders.GitHub == nil { + g.Config.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{} + } g.Config.Codesphere.GitProviders.GitHub.OAuth.ClientID = secret.Fields.Password } if secret, ok := secretsMap["githubAppsClientSecret"]; ok && secret.Fields != nil { + if g.Config.Codesphere.GitProviders == nil { + g.Config.Codesphere.GitProviders = &files.GitProvidersConfig{} + } + if g.Config.Codesphere.GitProviders.GitHub == nil { + g.Config.Codesphere.GitProviders.GitHub = &files.GitProviderConfig{} + } g.Config.Codesphere.GitProviders.GitHub.OAuth.ClientSecret = secret.Fields.Password } diff --git a/internal/installer/config_manager_profile.go b/internal/installer/config_manager_profile.go index 8844c130..3c8efffa 100644 --- a/internal/installer/config_manager_profile.go +++ b/internal/installer/config_manager_profile.go @@ -176,11 +176,454 @@ func (g *InstallConfig) applyCommonProperties() { g.Config.ManagedServiceBackends.Postgres = &files.PgManagedServiceConfig{} } if g.Config.Codesphere.ManagedServices == nil { + pgBackups := &files.ManagedServiceBackups{ + ConfigSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "endpointUrl": map[string]any{ + "type": "string", + "format": "uri", + "description": `S3-compatible endpoint URL for the backup storage, e.g. "http://rgw-load-balancer.rook-ceph.svc.cluster.local"`, + }, + "destinationPath": map[string]any{ + "type": "string", + "format": "uri", + "description": `S3 bucket URI where backups are stored. Must use the s3:// scheme, e.g. "s3://backup-test"`, + }, + "accessKeyId": map[string]any{ + "type": "string", + "description": "S3 access key for authentication. The associated user must have write access to the destination bucket.", + }, + }, + "required": []string{"endpointUrl", "destinationPath", "accessKeyId"}, + "additionalProperties": false, + }, + SecretsSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "secretKey": map[string]any{ + "type": "string", + "format": "password", + "description": "S3 secret key for authentication", + }, + }, + "required": []string{"secretKey"}, + "additionalProperties": false, + }, + } + postgresPlans := []files.ServicePlan{ + { + ID: 0, + Name: "Small", + Description: "0.5 vCPU / 500 MB Memory", + Parameters: map[string]files.PlanParam{ + "storage": {PricedAs: "storage-mib", Schema: map[string]interface{}{"type": "integer", "default": 10240, "readOnly": false, "minimum": 512, "x-update-constraint": "increase-only"}}, + "cpu": {PricedAs: "cpu-tenths", Schema: map[string]interface{}{"type": "number", "default": 5, "readOnly": true}}, + "memory": {PricedAs: "ram-mib", Schema: map[string]interface{}{"type": "integer", "default": 512, "readOnly": true}}, + }, + }, + { + ID: 1, + Name: "Medium", + Description: "1 vCPU / 1 GB Memory", + Parameters: map[string]files.PlanParam{ + "storage": {PricedAs: "storage-mib", Schema: map[string]interface{}{"type": "integer", "default": 25600, "readOnly": false, "minimum": 512}}, + "cpu": {PricedAs: "cpu-tenths", Schema: map[string]interface{}{"type": "number", "default": 10, "readOnly": true}}, + "memory": {PricedAs: "ram-mib", Schema: map[string]interface{}{"type": "integer", "default": 1024, "readOnly": true}}, + }, + }, + { + ID: 2, + Name: "Medium High-Mem", + Description: "1 vCPU / 2 GB Memory", + Parameters: map[string]files.PlanParam{ + "storage": {PricedAs: "storage-mib", Schema: map[string]interface{}{"type": "integer", "default": 25600, "readOnly": false, "minimum": 512}}, + "cpu": {PricedAs: "cpu-tenths", Schema: map[string]interface{}{"type": "number", "default": 10, "readOnly": true}}, + "memory": {PricedAs: "ram-mib", Schema: map[string]interface{}{"type": "integer", "default": 2048, "readOnly": true}}, + }, + }, + { + ID: 3, + Name: "Large", + Description: "2 vCPU / 4 GB Memory", + Parameters: map[string]files.PlanParam{ + "storage": {PricedAs: "storage-mib", Schema: map[string]interface{}{"type": "integer", "default": 51200, "readOnly": false, "minimum": 512}}, + "cpu": {PricedAs: "cpu-tenths", Schema: map[string]interface{}{"type": "number", "default": 20, "readOnly": true}}, + "memory": {PricedAs: "ram-mib", Schema: map[string]interface{}{"type": "integer", "default": 4096, "readOnly": true}}, + }, + }, + { + ID: 4, + Name: "Extra Large", + Description: "4 vCPU / 8 GB Memory", + Parameters: map[string]files.PlanParam{ + "storage": {PricedAs: "storage-mib", Schema: map[string]interface{}{"type": "integer", "default": 153600, "readOnly": false, "minimum": 512}}, + "cpu": {PricedAs: "cpu-tenths", Schema: map[string]interface{}{"type": "number", "default": 40, "readOnly": true}}, + "memory": {PricedAs: "ram-mib", Schema: map[string]interface{}{"type": "integer", "default": 8192, "readOnly": true}}, + }, + }, + } + ferretDbPlans := make([]files.ServicePlan, len(postgresPlans)) + for i, plan := range postgresPlans { + params := make(map[string]files.PlanParam, len(plan.Parameters)+2) + for k, v := range plan.Parameters { + params[k] = v + } + params["ferretdbCpu"] = files.PlanParam{ + PricedAs: "cpu-tenths", + Schema: map[string]interface{}{"type": "number", "default": 3, "readOnly": false, "minimum": 1, "maximum": 10}, + } + params["ferretdbMemory"] = files.PlanParam{ + PricedAs: "ram-mib", + Schema: map[string]interface{}{"type": "integer", "default": 128, "readOnly": false, "minimum": 128, "maximum": 1024}, + } + ferretDbPlans[i] = files.ServicePlan{ID: plan.ID, Name: plan.Name, Description: plan.Description, Parameters: params} + } g.Config.Codesphere.ManagedServices = []files.ManagedServiceConfig{ - {Name: "postgres", Version: "v1"}, - {Name: "babelfish", Version: "v1"}, - {Name: "s3", Version: "v1"}, - {Name: "virtual-k8s", Version: "v1"}, + { + Name: "postgres", + Version: "v1", + Author: "Codesphere", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{ + Endpoint: "http://ms-backend-postgres.postgres-operator:3000/api/v1/postgres", + }, + }, + Category: "Database", + Description: "Open-source database system tailored for efficient data management and scalability. Newest version of the Provider using the Cloud-Native Operator", + DisplayName: "PostgreSQL", + IconURL: "/ide/assets/managed-services/postgresql.svg", + Scope: "global", + Capabilities: &files.ManagedServiceCapabilities{ + Pause: true, + Backups: true, + PointInTimeRecovery: true, + }, + ConfigSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "version": map[string]interface{}{ + "type": "string", + "description": "Version of the Postgres DB. Includes pre-installed extensions compatible with this version. Extension versions are managed and cannot be customized.", + "enum": []string{"17.9", "17.6", "16.13", "16.10", "15.17", "15.14", "14.22", "14.19"}, + "default": "17.9", + "readOnly": false, + "x-update-constraint": "minor-upgrade-only", + }, + "userName": map[string]interface{}{ + "type": "string", + "default": "app", + "pattern": "^(?!postgres$)", + "description": `Cannot be "postgres" (reserved for the superuser).`, + "x-update-constraint": "immutable", + }, + "databaseName": map[string]interface{}{ + "type": "string", + "default": "app", + "x-update-constraint": "immutable", + }, + }, + "required": []string{}, + "additionalProperties": false, + }, + DetailsSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "port": map[string]interface{}{"type": "integer"}, + "hostname": map[string]interface{}{"type": "string", "format": "hostname"}, + "dsn": map[string]interface{}{"type": "string", "format": "uri"}, + "ready": map[string]interface{}{"type": "boolean"}, + }, + "required": []string{"port", "hostname", "dsn", "ready"}, + "additionalProperties": false, + }, + SecretsSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "userPassword": map[string]interface{}{"type": "string", "format": "password", "x-update-constraint": "immutable"}, + "superuserPassword": map[string]interface{}{"type": "string", "format": "password"}, + }, + "required": []string{"userPassword", "superuserPassword"}, + "additionalProperties": false, + }, + Backups: pgBackups, + Plans: postgresPlans, + }, + { + Name: "babelfish", + Version: "v1", + Author: "Codesphere", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{ + Endpoint: "http://ms-backend-postgres.postgres-operator:3000/api/v1/babelfish", + }, + }, + Category: "Database", + Description: "PostgreSQL instance with Babelfish extension to support applications requiring Microsoft TDS compatibility", + DisplayName: "Babelfish (T-SQL compatible)", + IconURL: "https://codesphere.com/ide/assets/managed-services/babelfish.svg", + Scope: "global", + Capabilities: &files.ManagedServiceCapabilities{ + Pause: true, + Backups: true, + PointInTimeRecovery: true, + }, + ConfigSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "version": map[string]interface{}{ + "type": "string", + "description": "Version of the Postgres DB and the corresponding version of Babelfish", + "enum": []string{"17.6-5.3.0", "16.10-4.7.0"}, + "default": "17.6-5.3.0", + "readOnly": false, + "x-update-constraint": "minor-upgrade-only", + }, + }, + "required": []string{}, + "additionalProperties": false, + }, + DetailsSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "port": map[string]interface{}{"type": "integer"}, + "hostname": map[string]interface{}{"type": "string", "format": "hostname"}, + "dsn": map[string]interface{}{"type": "string", "format": "uri", "description": "TDS connection string for the superuser and master database. Use this to connect with full administrative privileges."}, + "ready": map[string]interface{}{"type": "boolean"}, + }, + "required": []string{"port", "hostname", "dsn", "ready"}, + "additionalProperties": false, + }, + SecretsSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "superuserPassword": map[string]interface{}{"type": "string", "format": "password"}, + }, + "required": []string{"superuserPassword"}, + "additionalProperties": false, + }, + Backups: pgBackups, + Plans: postgresPlans, + }, + { + Name: "ferretdb", + Version: "v0", + Author: "Codesphere", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{ + Endpoint: "http://ms-backend-postgres.postgres-operator:3000/api/v1/ferretdb", + }, + }, + Category: "Database", + Description: "FerretDB based provider for MongoDB-compatible document database workloads. Powered by PostgreSQL.", + DisplayName: "Codesphere Document DB (MongoDB compatible)", + IconURL: "/ide/assets/managed-services/ferretdb.svg", + Scope: "global", + Capabilities: &files.ManagedServiceCapabilities{ + Pause: true, + }, + ConfigSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "version": map[string]interface{}{ + "type": "string", + "description": "Version of the Postgres / DocumentDB extension / FerretDB", + "enum": []string{"17-0.107.0-ferretdb-2.7.0"}, + "default": "17-0.107.0-ferretdb-2.7.0", + "readOnly": false, + "x-update-constraint": "minor-upgrade-only", + }, + }, + "required": []string{}, + "additionalProperties": false, + }, + DetailsSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "port": map[string]interface{}{"type": "integer"}, + "hostname": map[string]interface{}{"type": "string", "format": "hostname"}, + "dsn": map[string]interface{}{"type": "string", "format": "uri", "description": "MongoDB connection string for the admin user and admin database. Use this to connect with full administrative privileges."}, + "ready": map[string]interface{}{"type": "boolean"}, + }, + "required": []string{"port", "hostname", "dsn", "ready"}, + "additionalProperties": false, + }, + SecretsSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "superuserPassword": map[string]interface{}{"type": "string", "format": "password"}, + }, + "required": []string{"superuserPassword"}, + "additionalProperties": false, + }, + Plans: ferretDbPlans, + }, + { + Name: "s3", + Version: "v1", + Author: "Codesphere", + Backend: files.ManagedServiceBackend{ + API: files.ManagedServiceAPI{ + Endpoint: "http://ms-backend-s3.rook-ceph:3000/api/v1/s3", + }, + }, + Category: "Storage", + Description: "S3-compatible object storage for persisting unstructured data artifacts", + DisplayName: "Object Storage", + IconURL: "/ide/assets/managed-services/s3-bucket.svg", + Scope: "global", + Capabilities: &files.ManagedServiceCapabilities{ + Pause: false, + Backups: false, + PointInTimeRecovery: false, + }, + Backups: &files.ManagedServiceBackups{ + ConfigSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "endpointUrl": map[string]any{ + "type": "string", + "format": "uri", + "description": `S3-compatible endpoint URL for the backup storage, e.g. "http://rgw-load-balancer.rook-ceph.svc.cluster.local"`, + }, + "accessKeyId": map[string]any{ + "type": "string", + "description": "S3 access key for authentication at the backup store.", + }, + "path": map[string]any{ + "type": "string", + "description": `S3 path (bucket name with optional subpath), without s3://, e.g. "my-bucket/backups"`, + }, + }, + "required": []string{"endpointUrl", "accessKeyId", "path"}, + "additionalProperties": false, + }, + SecretsSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "secretKey": map[string]any{ + "type": "string", + "format": "password", + "description": "S3 secret key for authentication at the backup store", + }, + }, + "required": []string{"secretKey"}, + "additionalProperties": false, + }, + }, + ConfigSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "accessKey": map[string]interface{}{ + "type": "string", + "pattern": "^[A-Z0-9]{20}$", + "description": "Has to be cluster-unique. Exactly 20 uppercase letters (A-Z) or digits (0-9).", + "x-update-constraint": "immutable", + }, + "userDisplayName": map[string]interface{}{ + "type": "string", + "readOnly": false, + "default": "My S3 User", + "x-update-constraint": "immutable", + }, + "initialBucketName": map[string]interface{}{ + "type": "string", + "pattern": `^(?!\.)(?!-)(?!.*\.-)(?!.*-\.)(?!.*\.\.)[a-z0-9][a-z0-9.-]{2,62}(? yaml.Marshal uses struct field tags and does NOT + // call this Marshal() method. + data, err := yaml.Marshal(c) + if err != nil { + return nil, err + } + + // Parse into a node tree so we can swap out the codesphere value. + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil { + return nil, err + } + + if err := replaceYAMLMappingValue(&root, "codesphere", c.CodesphereConfigPath); err != nil { + return nil, fmt.Errorf("failed to set codesphere path reference: %w", err) + } + + return yaml.Marshal(&root) +} + +// replaceYAMLMappingValue replaces the value of a top-level mapping key with a plain string scalar. +func replaceYAMLMappingValue(root *yaml.Node, key, value string) error { + mapping := root + if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { + mapping = root.Content[0] + } + if mapping.Kind != yaml.MappingNode { + return fmt.Errorf("expected a YAML mapping node, got kind %d", mapping.Kind) + } + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + mapping.Content[i+1] = &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: value, + } + return nil + } + } + return fmt.Errorf("key %q not found in YAML mapping", key) } -// Unmarshal deserializes YAML data into the RootConfig +// Unmarshal deserializes YAML data into the RootConfig. func (c *RootConfig) Unmarshal(data []byte) error { - return yaml.Unmarshal(data, c) + // Parse the document into a raw node first so we can inspect the codesphere field. + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return err + } + if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { + mapping := doc.Content[0] + if mapping.Kind == yaml.MappingNode { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == "codesphere" && mapping.Content[i+1].Kind == yaml.ScalarNode { + c.CodesphereConfigPath = mapping.Content[i+1].Value + mapping.Content = append(mapping.Content[:i], mapping.Content[i+2:]...) + break + } + } + } + } + return doc.Decode(c) } func NewRootConfig() RootConfig { diff --git a/internal/installer/files/config_yaml_test.go b/internal/installer/files/config_yaml_test.go index 78cef43b..6cf7bfdb 100644 --- a/internal/installer/files/config_yaml_test.go +++ b/internal/installer/files/config_yaml_test.go @@ -188,6 +188,25 @@ codesphere: Expect(rootConfig.Registry.Server).To(Equal("minimal.registry.com")) Expect(rootConfig.Codesphere.DeployConfig.Images).To(BeEmpty()) }) + + It("should handle LTS 1.77.2 format where codesphere is a path string", func() { + lts177Yaml := `registry: + server: registry.example.com +codesphere: /etc/codesphere/codesphere.yaml +` + err := os.WriteFile(configFile, []byte(lts177Yaml), 0644) + Expect(err).NotTo(HaveOccurred()) + + data, err := os.ReadFile(configFile) + Expect(err).NotTo(HaveOccurred()) + + err = rootConfig.Unmarshal(data) + Expect(err).NotTo(HaveOccurred()) + + Expect(rootConfig.Registry.Server).To(Equal("registry.example.com")) + Expect(rootConfig.CodesphereConfigPath).To(Equal("/etc/codesphere/codesphere.yaml")) + Expect(rootConfig.Codesphere.DeployConfig.Images).To(BeEmpty()) + }) }) Describe("ExtractBomRefs", func() { diff --git a/internal/installer/k0sctl_config.go b/internal/installer/k0sctl_config.go index f41bf18a..a067c4e4 100644 --- a/internal/installer/k0sctl_config.go +++ b/internal/installer/k0sctl_config.go @@ -69,12 +69,20 @@ type K0sctlApplyHooks struct { } func createK0sctlHost(node files.K8sNode, role string, installFlags []string, sshKeyPath string, k0sBinaryPath string) K0sctlHost { + sshAddress := node.IPAddress + if node.SSHAddress != "" { + sshAddress = node.SSHAddress + } + sshPort := 22 + if node.SSHPort != 0 { + sshPort = node.SSHPort + } host := K0sctlHost{ Role: role, SSH: K0sctlSSH{ - Address: node.IPAddress, + Address: sshAddress, User: "root", - Port: 22, + Port: sshPort, KeyPath: sshKeyPath, }, InstallFlags: installFlags, diff --git a/internal/installer/node/node.go b/internal/installer/node/node.go index 69a8aea3..ed11a909 100644 --- a/internal/installer/node/node.go +++ b/internal/installer/node/node.go @@ -93,9 +93,12 @@ func (r *SSHNodeClient) RunCommand(n *Node, username string, command string) err _ = session.Setenv("OMS_PORTAL_API", os.Getenv("OMS_PORTAL_API")) _ = agent.RequestAgentForwarding(session) // Best effort, ignore errors + var stdoutBuf bytes.Buffer var stderrBuf bytes.Buffer - session.Stderr = &stderrBuf - if !r.Quiet { + if r.Quiet { + session.Stdout = &stdoutBuf + session.Stderr = &stderrBuf + } else { session.Stdout = os.Stdout session.Stderr = os.Stderr } @@ -106,8 +109,17 @@ func (r *SSHNodeClient) RunCommand(n *Node, username string, command string) err if err := session.Wait(); err != nil { // A non-zero exit status from the remote command is also considered an error - if r.Quiet && stderrBuf.Len() > 0 { - return fmt.Errorf("command failed: %w\n%s", err, stderrBuf.String()) + if r.Quiet { + var extra string + if stdoutBuf.Len() > 0 { + extra += "\nstdout:\n" + stdoutBuf.String() + } + if stderrBuf.Len() > 0 { + extra += "\nstderr:\n" + stderrBuf.String() + } + if extra != "" { + return fmt.Errorf("command failed: %w%s", err, extra) + } } return fmt.Errorf("command failed: %w", err) } diff --git a/internal/portal/portal.go b/internal/portal/portal.go index 7b337e7d..a368f90b 100644 --- a/internal/portal/portal.go +++ b/internal/portal/portal.go @@ -94,6 +94,10 @@ func (c *PortalClient) isOKResponseStatus(resp *http.Response) error { return errors.New("unauthorized: invalid API key") } + if resp.StatusCode == http.StatusTooManyRequests { + return fmt.Errorf("OMS-Portal rate limit exceeded (HTTP 429): please wait before retrying") + } + if resp.StatusCode >= 300 { log.Printf("Non-2xx response received from OMS-Portal (%s) - Status: %d", c.Env.GetOmsPortalApi(), resp.StatusCode) diff --git a/internal/portal/portal_test.go b/internal/portal/portal_test.go index 8797da67..b4444b86 100644 --- a/internal/portal/portal_test.go +++ b/internal/portal/portal_test.go @@ -115,6 +115,27 @@ var _ = Describe("PortalClient", func() { }) }) + Context("HTTP Request has Status: TooManyRequests", func() { + BeforeEach(func() { + mockHttpClient.EXPECT().Do(mock.Anything).RunAndReturn( + func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + }, nil + }) + }) + + It("returns a rate limit error without calling the health check", func() { + testRequest, err := http.NewRequest("GET", "fake", nil) + Expect(err).ToNot(HaveOccurred()) + + resp, err := client.AuthorizedHttpRequest(testRequest) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("rate limit exceeded")) + Expect(resp).To(BeNil()) + }) + }) + Context("HTTP Request has Non-OK Status", func() { BeforeEach(func() { mockEnv.EXPECT().GetOmsPortalApi().Return(apiUrl)