feat: 支持开源部署对象存储与迁移初始化

.github/workflows/offline-package.yml

@@ -8,6 +8,9 @@ on:
         required: false
         default: "29.0.4"
 
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
 jobs:
   package:
     strategy:

.gitignore

@@ -9,10 +9,18 @@ logs
 
 # Local build artifacts
 monkeycode-ai/
+backend/bin/
 .pnpm-store/
 frontend/.vite/
 
 # Electron (electron-builder)
 desktop/release/
 desktop/release-full/
 frontend/release/
+
+# tar
+*.tar
+*.tgz
+
+# Builder
+dist

backend/Installation/center/.env.example

@@ -0,0 +1,15 @@
+REMOTE_IP=
+NGINX_PORT=80
+POSTGRES_DB=monkeycode-ai
+POSTGRES_USER=monkeycode-ai
+POSTGRES_PASSWORD=
+REDIS_PASSWORD=
+CLICKHOUSE_DB=monkeycode-ai
+CLICKHOUSE_USER=monkeycode-ai
+CLICKHOUSE_PASSWORD=
+RUSTFS_ACCESS_KEY=
+RUSTFS_SECRET_KEY=
+TEAM_EMAIL=
+TEAM_NAME=MonkeyCode
+TEAM_PASSWORD=
+SUBNET_PREFIX=10.100.50

backend/Installation/center/install.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -eu
+
+DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "installer must run as root"
+  exit 1
+fi
+
+exec "$DIR/installer" center

backend/Installation/runner/docker-compose.yml

@@ -0,0 +1,19 @@
+name: monkeycode_runner
+services:
+  orchestrator:
+    image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/codingmatrix-orchestrator:alpha-latest
+    container_name: monkeycode-orchestrator
+    restart: always
+    labels:
+      - "com.centurylinklabs.watchtower.enable=true"
+    environment:
+      - ORCHESTRATOR_GRPC_URL=${GRPC_URL}
+      - ORCHESTRATOR_AUTH_TOKEN=${TOKEN}
+      - ORCHESTRATOR_TYPE=docker
+      - ORCHESTRATOR_LOG_LEVEL=debug
+      - ORCHESTRATOR_GRPC_PROXY=${GRPC_PROXY:-}
+      - WATCHTOWER_HTTP_API_TOKEN=${WATCHTOWER_API_TOKEN:-change-this-token}
+      - TZ=Asia/Shanghai
+    volumes:
+      - ./data:/app/data
+      - /var/run/docker.sock:/var/run/docker.sock

backend/Makefile

@@ -4,6 +4,8 @@ OUTPUT=type=docker,dest=$(HOME)/tmp/mcai_server.tar
 GOCACHE=/root/.cache/go-build
 GOMODCACHE?=/go/pkg/mod
 REGISTRY=chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode
+INSTALLER_OUT=bin/installer
+.PHONY: installer
 # make server PLATFORM= TAG= OUTPUT_SERVER= GOCACHE=
 image:
 	docker buildx build \
@@ -45,3 +47,14 @@ check-generate:
 
 migrate_sql:
 	migrate create -ext sql -dir migration -seq ${SEQ}
+
+installer:
+	mkdir -p ${INSTALLER_OUT}/x86_64 ${INSTALLER_OUT}/aarch64
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${INSTALLER_OUT}/x86_64/installer ./cmd/installer
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o ${INSTALLER_OUT}/aarch64/installer ./cmd/installer
+	@echo Installer: ${INSTALLER_OUT}/x86_64/installer
+	@echo Installer: ${INSTALLER_OUT}/aarch64/installer
+
+.PHONY: offline-package
+offline-package:
+	ARCH=$${ARCH:-amd64} ./scripts/build-offline-package.sh

backend/biz/host/usecase/host.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -198,21 +199,56 @@ func (h *HostUsecase) InstallScript(ctx context.Context, token *domain.InstallRe
 		return "", errcode.ErrInvalidInstallToken
 	}
 
-	tmp, err := template.New("install").Parse(string(templates.InstallTmpl))
+	tplName := "install"
+	tplContent := templates.InstallTmpl
+	if h.cfg.HostInstaller.Mode == "offline" {
+		tplName = "install_offline"
+		tplContent = templates.InstallOfflineTmpl
+	}
+
+	tmp, err := template.New(tplName).Parse(string(tplContent))
 	if err != nil {
 		return "", fmt.Errorf("failed to parse template %s", err)
 	}
 	buf := bytes.NewBuffer([]byte(""))
 	param := map[string]any{
-		"token":    token.Token,
-		"grpc_url": h.cfg.TaskFlow.GrpcURL,
+		"token":              token.Token,
+		"grpc_url":           h.cfg.TaskFlow.GrpcURL,
+		"base_url":           h.cfg.Server.BaseURL,
+		"installer_url":      h.installerURL(),
+		"docker_bundle_path": h.installerBundlePath("docker.tgz"),
+		"host_bundle_path":   h.hostBundlePath(),
 	}
 	if err := tmp.Execute(buf, param); err != nil {
 		return "", fmt.Errorf("failed to execute template %s", err)
 	}
 	return buf.String(), nil
 }
 
+func (h *HostUsecase) installerURL() string {
+	if h.cfg.Server.BaseURL == "" {
+		return ""
+	}
+	baseurl, err := url.Parse(h.cfg.Server.BaseURL)
+	if err != nil {
+		return ""
+	}
+	baseurl = baseurl.JoinPath(h.cfg.StaticFiles.RoutePrefix, "installer")
+	return strings.TrimRight(baseurl.String(), "/") + "/{{.arch}}/installer"
+}
+
+func (h *HostUsecase) hostBundlePath() string {
+	bundlePath := h.cfg.HostInstaller.BundlePath
+	if bundlePath == "" {
+		bundlePath = "installer/{{.arch}}/host.tgz"
+	}
+	return "/" + strings.Trim(h.cfg.StaticFiles.RoutePrefix, "/") + "/" + strings.TrimLeft(bundlePath, "/")
+}
+
+func (h *HostUsecase) installerBundlePath(name string) string {
+	return "/" + strings.Trim(h.cfg.StaticFiles.RoutePrefix, "/") + "/installer/{{.arch}}/" + name
+}
+
 // List implements domain.HostUsecase.
 func (h *HostUsecase) List(ctx context.Context, uid uuid.UUID) (*domain.HostListResp, error) {
 	user, err := h.userRepo.Get(ctx, uid)

backend/biz/host/usecase/host_test.go

@@ -4,19 +4,115 @@ import (
 	"context"
 	"io"
 	"log/slog"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/google/uuid"
 	_ "github.com/mattn/go-sqlite3"
+	"github.com/redis/go-redis/v9"
 
+	"github.com/chaitin/MonkeyCode/backend/config"
 	"github.com/chaitin/MonkeyCode/backend/consts"
 	"github.com/chaitin/MonkeyCode/backend/db"
 	"github.com/chaitin/MonkeyCode/backend/db/enttest"
 	"github.com/chaitin/MonkeyCode/backend/domain"
 	"github.com/chaitin/MonkeyCode/backend/pkg/taskflow"
 )
 
+func TestInstallScriptDefaultsToOnlineInstaller(t *testing.T) {
+	t.Parallel()
+
+	mr := miniredis.RunT(t)
+	rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+	t.Cleanup(func() { _ = rdb.Close() })
+
+	token := "install-token"
+	if err := rdb.Set(context.Background(), "host:token:"+token, "1", time.Minute).Err(); err != nil {
+		t.Fatal(err)
+	}
+	u := &HostUsecase{
+		cfg: &config.Config{
+			TaskFlow: config.TaskFlow{GrpcURL: "121.41.208.82:50443"},
+		},
+		redis: rdb,
+	}
+
+	script, err := u.InstallScript(context.Background(), &domain.InstallReq{Token: token})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(script, "release.baizhi.cloud/monkeycode/runner/$ARCH/installer") {
+		t.Fatalf("script missing online installer: %s", script)
+	}
+	if !strings.Contains(script, "--env GRPC_URL=121.41.208.82:50443") {
+		t.Fatalf("script missing grpc url: %s", script)
+	}
+	if strings.Contains(script, "install_docker_from_bundle") {
+		t.Fatalf("online script should not include offline installer: %s", script)
+	}
+}
+
+func TestInstallScriptUsesOfflineBundle(t *testing.T) {
+	t.Parallel()
+
+	mr := miniredis.RunT(t)
+	rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+	t.Cleanup(func() { _ = rdb.Close() })
+
+	token := "install-token"
+	if err := rdb.Set(context.Background(), "host:token:"+token, "1", time.Minute).Err(); err != nil {
+		t.Fatal(err)
+	}
+	u := &HostUsecase{
+		cfg: &config.Config{
+			Server: struct {
+				Addr    string `mapstructure:"addr"`
+				BaseURL string `mapstructure:"base_url"`
+			}{BaseURL: "http://monkeycode.local"},
+			TaskFlow: config.TaskFlow{GrpcURL: "121.41.208.82:50443"},
+			StaticFiles: config.StaticFilesConfig{
+				RoutePrefix: "/static",
+			},
+			HostInstaller: config.HostInstaller{
+				Mode:       "offline",
+				BundlePath: "installer/{{.arch}}/host.tgz",
+			},
+		},
+		redis: rdb,
+	}
+
+	script, err := u.InstallScript(context.Background(), &domain.InstallReq{Token: token})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(script, "GRPC_URL=\"121.41.208.82:50443\"") {
+		t.Fatalf("script missing grpc url: %s", script)
+	}
+	if !strings.Contains(script, "INSTALLER_URL=\"http://monkeycode.local/static/installer/{{.arch}}/installer\"") {
+		t.Fatalf("script missing installer url: %s", script)
+	}
+	if !strings.Contains(script, "BASE_URL=\"http://monkeycode.local\"") || !strings.Contains(script, "MCAI_BASE_URL=\"$BASE_URL\"") {
+		t.Fatalf("script missing base url: %s", script)
+	}
+	if !strings.Contains(script, "HOST_BUNDLE_PATH=\"/static/installer/{{.arch}}/host.tgz\"") || !strings.Contains(script, "HOST_BUNDLE_PATH=${HOST_BUNDLE_PATH//\\{\\{.arch\\}\\}/$ARCH}") || !strings.Contains(script, "MCAI_HOST_BUNDLE_PATH=\"$HOST_BUNDLE_PATH\"") {
+		t.Fatalf("script missing host bundle path: %s", script)
+	}
+	if !strings.Contains(script, "DOCKER_BUNDLE_PATH=\"/static/installer/{{.arch}}/docker.tgz\"") || !strings.Contains(script, "DOCKER_BUNDLE_PATH=${DOCKER_BUNDLE_PATH//\\{\\{.arch\\}\\}/$ARCH}") || !strings.Contains(script, "MCAI_DOCKER_BUNDLE_PATH=\"$DOCKER_BUNDLE_PATH\"") {
+		t.Fatalf("script missing docker bundle path: %s", script)
+	}
+	if !strings.Contains(script, "TOKEN=\"install-token\"") || !strings.Contains(script, "MCAI_HOST_TOKEN=\"$TOKEN\"") {
+		t.Fatalf("script missing host token: %s", script)
+	}
+	if strings.Contains(script, "docker load") || strings.Contains(script, "docker compose") {
+		t.Fatalf("bootstrap script should not install host directly: %s", script)
+	}
+	if strings.Contains(script, "release.baizhi.cloud") {
+		t.Fatalf("script should not download public installer: %s", script)
+	}
+}
+
 func TestHostUsecase_markRecycledTasksFinished(t *testing.T) {
 	t.Parallel()
 

backend/biz/host/usecase/publichost_test.go

@@ -64,30 +64,32 @@ func TestPickHostSelectsHostByRandomOffset(t *testing.T) {
 	}
 }
 
-func TestPickHostTreatsNonPositiveWeightsAsOne(t *testing.T) {
+func TestPickHostIgnoresNonPositiveWeights(t *testing.T) {
 	u := &PublicHostUsecase{
 		repo: &publicHostRepoStub{
 			hosts: []*db.Host{
 				{ID: "host-a", Hostname: "a", Weight: 0},
 				{ID: "host-b", Hostname: "b", Weight: -2},
+				{ID: "host-c", Hostname: "c", Weight: 1},
 			},
 		},
 		taskflow: &taskflowClientStub{
 			hoster: &publicHosterStub{
 				onlineMap: map[string]bool{
 					"host-a": true,
 					"host-b": true,
+					"host-c": true,
 				},
 			},
 		},
 	}
 
 	prevRandUint64n := randUint64n
 	randUint64n = func(n uint64) (uint64, error) {
-		if n != 2 {
-			t.Fatalf("rand limit = %d, want 2", n)
+		if n != 1 {
+			t.Fatalf("rand limit = %d, want 1", n)
 		}
-		return 1, nil
+		return 0, nil
 	}
 	t.Cleanup(func() {
 		randUint64n = prevRandUint64n
@@ -97,8 +99,8 @@ func TestPickHostTreatsNonPositiveWeightsAsOne(t *testing.T) {
 	if err != nil {
 		t.Fatalf("PickHost() error = %v", err)
 	}
-	if host.ID != "host-b" {
-		t.Fatalf("PickHost() = %q, want %q", host.ID, "host-b")
+	if host.ID != "host-c" {
+		t.Fatalf("PickHost() = %q, want %q", host.ID, "host-c")
 	}
 }