feat: add LTS 1.77.2 compatibility and enhance related functionalities

cli/cmd/init_install_config.go

@@ -351,7 +351,11 @@ func (c *InitInstallConfigCmd) updateConfigFromOpts(config *files.RootConfig) *f
 	if len(c.Opts.CephHosts) > 0 {
 		cephHosts := []files.CephHost{}
 		for _, hostCfg := range c.Opts.CephHosts {
-			cephHosts = append(cephHosts, files.CephHost(hostCfg))
+			cephHosts = append(cephHosts, files.CephHost{
+				Hostname:  hostCfg.Hostname,
+				IPAddress: hostCfg.IPAddress,
+				IsMaster:  hostCfg.IsMaster,
+			})
 		}
 		config.Ceph.Hosts = cephHosts
 	}

cli/cmd/update_dockerfile_test.go

@@ -139,17 +139,12 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockImageManager := system.NewMockImageManager(GinkgoT())
 			mockFileIO := util.NewMockFileIO(GinkgoT())
 
-			// Create a temporary file for the Dockerfile
-			tempFile, err := os.CreateTemp("", "dockerfile-test-*")
+			pr, pw, err := os.Pipe()
 			Expect(err).To(BeNil())
-			DeferCleanup(func() {
-				_ = tempFile.Close()
-				_ = os.Remove(tempFile.Name())
-			})
-			_, err = tempFile.WriteString(sampleDockerfileContent)
+			_, err = pw.WriteString(sampleDockerfileContent)
 			Expect(err).To(BeNil())
-			// Reset file position to beginning
-			_, _ = tempFile.Seek(0, 0)
+			Expect(pw.Close()).To(Succeed())
+			DeferCleanup(func() { _ = pr.Close() })
 
 			c.Opts.Dockerfile = "Dockerfile"
 			c.Opts.Baseimage = ""
@@ -160,7 +155,7 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockPackageManager.EXPECT().GetBaseimagePath("", false).Return("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar", nil)
 			mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar").Return(nil)
 			mockPackageManager.EXPECT().FileIO().Return(mockFileIO)
-			mockFileIO.EXPECT().Open("Dockerfile").Return(tempFile, nil)
+			mockFileIO.EXPECT().Open("Dockerfile").Return(pr, nil)
 			mockFileIO.EXPECT().WriteFile("Dockerfile", []byte("FROM ubuntu:24.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(errors.New("write failed"))
 
 			err = c.UpdateDockerfile(mockPackageManager, mockImageManager, []string{})
@@ -173,17 +168,12 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockImageManager := system.NewMockImageManager(GinkgoT())
 			mockFileIO := util.NewMockFileIO(GinkgoT())
 
-			// Create a temporary file for the Dockerfile
-			tempFile, err := os.CreateTemp("", "dockerfile-test-*")
+			pr, pw, err := os.Pipe()
 			Expect(err).To(BeNil())
-			DeferCleanup(func() {
-				_ = tempFile.Close()
-				_ = os.Remove(tempFile.Name())
-			})
-			_, err = tempFile.WriteString(sampleDockerfileContent)
+			_, err = pw.WriteString(sampleDockerfileContent)
 			Expect(err).To(BeNil())
-			// Reset file position to beginning
-			_, _ = tempFile.Seek(0, 0)
+			Expect(pw.Close()).To(Succeed())
+			DeferCleanup(func() { _ = pr.Close() })
 
 			c.Opts.Dockerfile = "Dockerfile"
 			c.Opts.Baseimage = ""
@@ -193,7 +183,7 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockPackageManager.EXPECT().GetFullImageTag("").Return("ubuntu:24.04", nil)
 			mockPackageManager.EXPECT().GetBaseimagePath("", false).Return("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar", nil)
 			mockPackageManager.EXPECT().FileIO().Return(mockFileIO)
-			mockFileIO.EXPECT().Open("Dockerfile").Return(tempFile, nil)
+			mockFileIO.EXPECT().Open("Dockerfile").Return(pr, nil)
 			mockFileIO.EXPECT().WriteFile("Dockerfile", []byte("FROM ubuntu:24.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(nil)
 			mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar").Return(nil)
 
@@ -206,17 +196,12 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockImageManager := system.NewMockImageManager(GinkgoT())
 			mockFileIO := util.NewMockFileIO(GinkgoT())
 
-			// Create a temporary file for the Dockerfile
-			tempFile, err := os.CreateTemp("", "dockerfile-test-*")
+			pr, pw, err := os.Pipe()
 			Expect(err).To(BeNil())
-			DeferCleanup(func() {
-				_ = tempFile.Close()
-				_ = os.Remove(tempFile.Name())
-			})
-			_, err = tempFile.WriteString(sampleDockerfileContent)
+			_, err = pw.WriteString(sampleDockerfileContent)
 			Expect(err).To(BeNil())
-			// Reset file position to beginning
-			_, _ = tempFile.Seek(0, 0)
+			Expect(pw.Close()).To(Succeed())
+			DeferCleanup(func() { _ = pr.Close() })
 
 			c.Opts.Dockerfile = "Dockerfile"
 			c.Opts.Baseimage = "workspace-agent-20.04.tar"
@@ -226,7 +211,7 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockPackageManager.EXPECT().GetFullImageTag("workspace-agent-20.04.tar").Return("ubuntu:20.04", nil)
 			mockPackageManager.EXPECT().GetBaseimagePath("workspace-agent-20.04.tar", true).Return("/test/workdir/deps/codesphere/images/workspace-agent-20.04.tar", nil)
 			mockPackageManager.EXPECT().FileIO().Return(mockFileIO)
-			mockFileIO.EXPECT().Open("Dockerfile").Return(tempFile, nil)
+			mockFileIO.EXPECT().Open("Dockerfile").Return(pr, nil)
 			mockFileIO.EXPECT().WriteFile("Dockerfile", []byte("FROM ubuntu:20.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(nil)
 			mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-20.04.tar").Return(nil)
 
@@ -239,17 +224,12 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockImageManager := system.NewMockImageManager(GinkgoT())
 			mockFileIO := util.NewMockFileIO(GinkgoT())
 
-			// Create a temporary file for the Dockerfile
-			tempFile, err := os.CreateTemp("", "dockerfile-test-*")
+			pr, pw, err := os.Pipe()
 			Expect(err).To(BeNil())
-			DeferCleanup(func() {
-				_ = tempFile.Close()
-				_ = os.Remove(tempFile.Name())
-			})
-			_, err = tempFile.WriteString(sampleDockerfileContent)
+			_, err = pw.WriteString(sampleDockerfileContent)
 			Expect(err).To(BeNil())
-			// Reset file position to beginning
-			_, _ = tempFile.Seek(0, 0)
+			Expect(pw.Close()).To(Succeed())
+			DeferCleanup(func() { _ = pr.Close() })
 
 			c.Opts.Dockerfile = "custom/Dockerfile"
 			c.Opts.Baseimage = "workspace-agent-24.04.tar"
@@ -259,7 +239,7 @@ var _ = Describe("UpdateDockerfileCmd", func() {
 			mockPackageManager.EXPECT().GetFullImageTag("workspace-agent-24.04.tar").Return("registry.example.com/workspace-agent:24.04", nil)
 			mockPackageManager.EXPECT().GetBaseimagePath("workspace-agent-24.04.tar", false).Return("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar", nil)
 			mockPackageManager.EXPECT().FileIO().Return(mockFileIO)
-			mockFileIO.EXPECT().Open("custom/Dockerfile").Return(tempFile, nil)
+			mockFileIO.EXPECT().Open("custom/Dockerfile").Return(pr, nil)
 			mockFileIO.EXPECT().WriteFile("custom/Dockerfile", []byte("FROM registry.example.com/workspace-agent:24.04\nRUN apt-get update && apt-get install -y curl\nWORKDIR /app\nCOPY . .\nCMD [\"./start.sh\"]"), os.FileMode(0644)).Return(nil)
 			mockImageManager.EXPECT().LoadImage("/test/workdir/deps/codesphere/images/workspace-agent-24.04.tar").Return(nil)
 

docs/oms_beta_bootstrap-gcp.md

@@ -31,7 +31,7 @@ oms beta bootstrap-gcp [flags]
       --datacenter-name string                  Datacenter name (default: dev) (default "dev")
       --dns-project-id string                   GCP Project ID for Cloud DNS (optional)
       --dns-zone-name string                    Cloud DNS Zone Name (optional) (default "oms-testing")
-      --experiments stringArray                 Experiments to enable in Codesphere installation (optional) (default [managed-services,headless-services,vcluster,custom-service-image,ms-in-ls,secret-management,sub-path-mount])
+      --experiments stringArray                 Experiments to enable in Codesphere installation (optional) (default [managed-services,custom-service-image,ms-in-ls,secret-management,sub-path-mount])
       --external-loki-endpoint string           External Loki endpoint for Grafana Alloy log forwarding (optional)
       --external-loki-secret string             External Loki password stored in the generated vault (optional)
       --external-loki-user string               External Loki username for Grafana Alloy log forwarding (optional)

docs/oms_beta_bootstrap-local.md

@@ -17,7 +17,7 @@ oms beta bootstrap-local [flags]
 
 ```
       --base-domain string          Base domain for Codesphere (default "cs.local")
-      --experiments stringArray     Experiments to enable in Codesphere installation (optional) (default [managed-services,headless-services,vcluster,custom-service-image,ms-in-ls,secret-management,sub-path-mount])
+      --experiments stringArray     Experiments to enable in Codesphere installation (optional) (default [managed-services,custom-service-image,ms-in-ls,secret-management,sub-path-mount])
       --feature-flags stringArray   Feature flags to enable in Codesphere installation (optional)
   -h, --help                        help for bootstrap-local
       --install-config string       Path to install config file (default: <install-dir>/config.yaml)

internal/bootstrap/gcp/gcp.go

@@ -62,8 +62,6 @@ func GetDNSRecordNames(baseDomain string) []struct {
 
 var DefaultExperiments []string = []string{
 	"managed-services",
-	"headless-services",
-	"vcluster",
 	"custom-service-image",
 	"ms-in-ls",
 	"secret-management",
@@ -80,9 +78,10 @@ type GCPBootstrapper struct {
 	// Environment
 	Env *CodesphereEnvironment
 	// SSH command runner
-	NodeClient   node.NodeClient
-	PortalClient portal.Portal
-	GitHubClient github.GitHubClient
+	NodeClient       node.NodeClient
+	PortalClient     portal.Portal
+	GitHubClient     github.GitHubClient
+	OmsBinaryBuilder func() (string, func(), error)
 }
 
 type CodesphereEnvironment struct {
@@ -182,16 +181,17 @@ func NewGCPBootstrapper(
 	gitHubClient github.GitHubClient,
 ) (*GCPBootstrapper, error) {
 	return &GCPBootstrapper{
-		ctx:          ctx,
-		stlog:        stlog,
-		fw:           fw,
-		icg:          icg,
-		GCPClient:    gcpClient,
-		Env:          CodesphereEnv,
-		NodeClient:   sshRunner,
-		PortalClient: portalClient,
-		Time:         time,
-		GitHubClient: gitHubClient,
+		ctx:              ctx,
+		stlog:            stlog,
+		fw:               fw,
+		icg:              icg,
+		GCPClient:        gcpClient,
+		Env:              CodesphereEnv,
+		NodeClient:       sshRunner,
+		PortalClient:     portalClient,
+		Time:             time,
+		GitHubClient:     gitHubClient,
+		OmsBinaryBuilder: BuildOmsLinuxBinary,
 	}, nil
 }
 
@@ -922,6 +922,16 @@ func (b *GCPBootstrapper) InstallCodesphere() error {
 		return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err)
 	}
 
+	if ltsSpec := FindLTSSpec(b.Env.InstallVersion); ltsSpec != nil {
+		if ltsSpec.RequiresOmsBinaryUpdate {
+			if err := b.ensureNewOmsBinaryOnJumpbox(); err != nil {
+				return fmt.Errorf("failed to update OMS binary on jumpbox for %s: %w", b.Env.InstallVersion, err)
+			}
+		}
+		b.startLTSCephMasterWatcher()
+		defer b.stopLTSCephMasterWatcher()
+	}
+
 	err = b.runInstallCommand(fullPackageFilename)
 	if err != nil {
 		return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)
@@ -930,6 +940,61 @@ func (b *GCPBootstrapper) InstallCodesphere() error {
 	return nil
 }
 
+// ensureNewOmsBinaryOnJumpbox copies a freshly-built linux/amd64 OMS binary to
+// the jumpbox, replacing the old installed version.
+func (b *GCPBootstrapper) ensureNewOmsBinaryOnJumpbox() error {
+	b.stlog.Logf("Updating OMS binary on jumpbox for %s compatibility...", b.Env.InstallVersion)
+
+	binaryPath, cleanup, err := b.OmsBinaryBuilder()
+	if err != nil {
+		return fmt.Errorf("failed to prepare OMS linux binary: %w", err)
+	}
+	defer cleanup()
+
+	const remoteTmpPath = "/tmp/oms-new"
+	if err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, binaryPath, remoteTmpPath); err != nil {
+		return fmt.Errorf("failed to copy OMS binary to jumpbox: %w", err)
+	}
+
+	if err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("chmod +x %s && mv %s /usr/local/bin/oms", remoteTmpPath, remoteTmpPath)); err != nil {
+		return fmt.Errorf("failed to install OMS binary on jumpbox: %w", err)
+	}
+
+	return nil
+}
+
+// startLTSCephMasterWatcher starts a background process on the ceph master node that continuously
+// re-adds the master to the Ceph orchestrator host inventory. This is required for LTS versions
+// because the installer's configureHosts step applies a declarative host spec containing only the
+// non-master nodes, which removes the master from the inventory. The watcher restores it within
+// seconds, before the subsequent configureMonitors step runs.
+func (b *GCPBootstrapper) startLTSCephMasterWatcher() {
+	if len(b.Env.CephNodes) == 0 || len(b.Env.InstallConfig.Ceph.Hosts) == 0 {
+		return
+	}
+	masterHost := b.Env.InstallConfig.Ceph.Hosts[0]
+	// Use cephadm shell (same as the installer) so the command runs inside the ceph container,
+	// bypassing any standalone-binary or keyring availability issues on the host.
+	// The FSID is auto-detected from /var/lib/ceph/; all output is logged for diagnostics.
+	cmd := fmt.Sprintf(
+		`nohup bash -c "while true; do FSID=\$(ls /var/lib/ceph/ 2>/dev/null | head -1); [ -n \"\$FSID\" ] && [ -x /usr/local/bin/cephadm ] && /usr/local/bin/cephadm shell --fsid \"\$FSID\" -- ceph orch host add %s %s 2>&1; sleep 3; done" > /tmp/ceph-host-watcher.log 2>&1 & echo $! > /tmp/ceph-host-watcher.pid`,
+		masterHost.Hostname,
+		masterHost.IPAddress,
+	)
+	if err := b.Env.CephNodes[0].RunSSHCommand("root", cmd); err != nil {
+		b.stlog.Logf("Note: could not start ceph master host watcher on %s: %v", masterHost.Hostname, err)
+	}
+}
+
+// stopLTSCephMasterWatcher stops the background watcher started by startLTSCephMasterWatcher.
+func (b *GCPBootstrapper) stopLTSCephMasterWatcher() {
+	if len(b.Env.CephNodes) == 0 || len(b.Env.InstallConfig.Ceph.Hosts) == 0 {
+		return
+	}
+	cmd := `kill $(cat /tmp/ceph-host-watcher.pid 2>/dev/null) 2>/dev/null; rm -f /tmp/ceph-host-watcher.pid /tmp/ceph-host-watcher.log`
+	_ = b.Env.CephNodes[0].RunSSHCommand("root", cmd)
+}
+
 func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) {
 	packageFilename := "installer.tar.gz"
 	if b.Env.RegistryType == RegistryTypeGitHub {
@@ -959,7 +1024,13 @@ func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) {
 		packageFilename, b.Env.InstallHash, b.Env.InstallVersion)
 	err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd)
 	if err != nil {
-		return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
+		// A stale partial file from a previous (different) build can cause MD5 verification to
+		// fail even after a successful byte-range resume. Delete it and retry from scratch.
+		b.stlog.Logf("Download failed; removing any stale partial file and retrying from scratch...")
+		cleanAndRetryCmd := fmt.Sprintf("rm -f %s && %s", fullPackageFilename, downloadCmd)
+		if retryErr := b.Env.Jumpbox.RunSSHCommand("root", cleanAndRetryCmd); retryErr != nil {
+			return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", retryErr)
+		}
 	}
 
 	return fullPackageFilename, nil

internal/bootstrap/gcp/gcp_test.go

@@ -263,7 +263,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 				Expect(cpNode.GetInternalIP()).To(Equal("10.0.0.1"))
 			}
 
-			Expect(len(bs.Env.InstallConfig.Codesphere.ManagedServices)).To(Equal(4))
+			Expect(len(bs.Env.InstallConfig.Codesphere.ManagedServices)).To(Equal(5))
 		})
 	})
 
@@ -1227,6 +1227,55 @@ var _ = Describe("GCP Bootstrapper", func() {
 				Expect(err).NotTo(HaveOccurred())
 			})
 
+			Context("LTS 1.77.2", func() {
+				BeforeEach(func() {
+					csEnv.InstallVersion = "codesphere-lts-v1.77.2"
+				})
+				JustBeforeEach(func() {
+					// Inject a stub binary builder so tests don't invoke `go build`.
+					bs.OmsBinaryBuilder = func() (string, func(), error) {
+						f, err := os.CreateTemp("", "oms-test-binary-*")
+						Expect(err).NotTo(HaveOccurred())
+						Expect(f.Close()).To(Succeed())
+						return f.Name(), func() { Expect(os.Remove(f.Name())).To(Succeed()) }, nil
+					}
+				})
+				It("downloads package, updates OMS binary, and installs codesphere", func() {
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+						"oms download package -f installer.tar.gz -H abc1234567890 codesphere-lts-v1.77.2").Return(nil)
+					nodeClient.EXPECT().CopyFile(mock.MatchedBy(jumpboxMatcher), mock.Anything, "/tmp/oms-new").Return(nil)
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+						"chmod +x /tmp/oms-new && mv /tmp/oms-new /usr/local/bin/oms").Return(nil)
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+						"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p codesphere-lts-v1.77.2-abc1234567890-installer.tar.gz").Return(nil)
+
+					err := bs.InstallCodesphere()
+					Expect(err).NotTo(HaveOccurred())
+				})
+
+				It("fails when OmsBinaryBuilder fails", func() {
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+						"oms download package -f installer.tar.gz -H abc1234567890 codesphere-lts-v1.77.2").Return(nil)
+					bs.OmsBinaryBuilder = func() (string, func(), error) {
+						return "", func() {}, fmt.Errorf("build failed")
+					}
+
+					err := bs.InstallCodesphere()
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(ContainSubstring("failed to update OMS binary on jumpbox for codesphere-lts-v1.77.2"))
+				})
+
+				It("fails when copying binary to jumpbox fails", func() {
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+						"oms download package -f installer.tar.gz -H abc1234567890 codesphere-lts-v1.77.2").Return(nil)
+					nodeClient.EXPECT().CopyFile(mock.MatchedBy(jumpboxMatcher), mock.Anything, "/tmp/oms-new").Return(fmt.Errorf("copy failed"))
+
+					err := bs.InstallCodesphere()
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(ContainSubstring("failed to update OMS binary on jumpbox for codesphere-lts-v1.77.2"))
+				})
+			})
+
 			Context("with local package", func() {
 				BeforeEach(func() {
 					csEnv.InstallLocal = "fake-installer-lite.tar.gz"
@@ -1288,8 +1337,23 @@ var _ = Describe("GCP Bootstrapper", func() {
 				})
 			})
 
+			It("retries with a clean slate when download fails, succeeds on retry", func() {
+				downloadCmd := "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3"
+				cleanAndRetryCmd := "rm -f v1.2.3-abc1234567890-installer.tar.gz && " + downloadCmd
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", downloadCmd).Return(fmt.Errorf("md5 mismatch")).Once()
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", cleanAndRetryCmd).Return(nil).Once()
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+					"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-abc1234567890-installer.tar.gz").Return(nil).Once()
+
+				err := bs.InstallCodesphere()
+				Expect(err).NotTo(HaveOccurred())
+			})
+
 			It("fails when download package fails", func() {
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(fmt.Errorf("download error"))
+				downloadCmd := "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3"
+				cleanAndRetryCmd := "rm -f v1.2.3-abc1234567890-installer.tar.gz && " + downloadCmd
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", downloadCmd).Return(fmt.Errorf("download error")).Once()
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", cleanAndRetryCmd).Return(fmt.Errorf("download error on retry")).Once()
 
 				err := bs.InstallCodesphere()
 				Expect(err).To(HaveOccurred())