diff --git a/.github/workflows/offline-package.yml b/.github/workflows/offline-package.yml index c97ccfda..df2271b0 100644 --- a/.github/workflows/offline-package.yml +++ b/.github/workflows/offline-package.yml @@ -8,6 +8,9 @@ on: required: false default: "29.0.4" +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: package: strategy: diff --git a/.gitignore b/.gitignore index 307712be..76acdd03 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ logs # Local build artifacts monkeycode-ai/ +backend/bin/ .pnpm-store/ frontend/.vite/ @@ -16,3 +17,10 @@ frontend/.vite/ desktop/release/ desktop/release-full/ frontend/release/ + +# tar +*.tar +*.tgz + +# Builder +dist \ No newline at end of file diff --git a/backend/Installation/center/.env.example b/backend/Installation/center/.env.example new file mode 100644 index 00000000..932382c2 --- /dev/null +++ b/backend/Installation/center/.env.example @@ -0,0 +1,15 @@ +REMOTE_IP= +NGINX_PORT=80 +POSTGRES_DB=monkeycode-ai +POSTGRES_USER=monkeycode-ai +POSTGRES_PASSWORD= +REDIS_PASSWORD= +CLICKHOUSE_DB=monkeycode-ai +CLICKHOUSE_USER=monkeycode-ai +CLICKHOUSE_PASSWORD= +RUSTFS_ACCESS_KEY= +RUSTFS_SECRET_KEY= +TEAM_EMAIL= +TEAM_NAME=MonkeyCode +TEAM_PASSWORD= +SUBNET_PREFIX=10.100.50 diff --git a/backend/Installation/center/install.sh b/backend/Installation/center/install.sh new file mode 100644 index 00000000..6d413b10 --- /dev/null +++ b/backend/Installation/center/install.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +if [ "$(id -u)" -ne 0 ]; then + echo "installer must run as root" + exit 1 +fi + +exec "$DIR/installer" center diff --git a/backend/Installation/runner/docker-compose.yml b/backend/Installation/runner/docker-compose.yml new file mode 100644 index 00000000..662a7f66 --- /dev/null +++ b/backend/Installation/runner/docker-compose.yml @@ -0,0 +1,19 @@ +name: monkeycode_runner +services: + orchestrator: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/codingmatrix-orchestrator:alpha-latest + container_name: monkeycode-orchestrator + restart: always + labels: + - "com.centurylinklabs.watchtower.enable=true" + environment: + - ORCHESTRATOR_GRPC_URL=${GRPC_URL} + - ORCHESTRATOR_AUTH_TOKEN=${TOKEN} + - ORCHESTRATOR_TYPE=docker + - ORCHESTRATOR_LOG_LEVEL=debug + - ORCHESTRATOR_GRPC_PROXY=${GRPC_PROXY:-} + - WATCHTOWER_HTTP_API_TOKEN=${WATCHTOWER_API_TOKEN:-change-this-token} + - TZ=Asia/Shanghai + volumes: + - ./data:/app/data + - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file diff --git a/backend/Makefile b/backend/Makefile index c2c93c8c..efa409cc 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -4,6 +4,8 @@ OUTPUT=type=docker,dest=$(HOME)/tmp/mcai_server.tar GOCACHE=/root/.cache/go-build GOMODCACHE?=/go/pkg/mod REGISTRY=chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode +INSTALLER_OUT=bin/installer +.PHONY: installer # make server PLATFORM= TAG= OUTPUT_SERVER= GOCACHE= image: docker buildx build \ @@ -45,3 +47,14 @@ check-generate: migrate_sql: migrate create -ext sql -dir migration -seq ${SEQ} + +installer: + mkdir -p ${INSTALLER_OUT}/x86_64 ${INSTALLER_OUT}/aarch64 + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${INSTALLER_OUT}/x86_64/installer ./cmd/installer + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o ${INSTALLER_OUT}/aarch64/installer ./cmd/installer + @echo Installer: ${INSTALLER_OUT}/x86_64/installer + @echo Installer: ${INSTALLER_OUT}/aarch64/installer + +.PHONY: offline-package +offline-package: + ARCH=$${ARCH:-amd64} ./scripts/build-offline-package.sh diff --git a/backend/biz/host/usecase/host.go b/backend/biz/host/usecase/host.go index 2320cd21..ce1d2b10 100644 --- a/backend/biz/host/usecase/host.go +++ b/backend/biz/host/usecase/host.go @@ -11,6 +11,7 @@ import ( "net/url" "sort" "strconv" + "strings" "time" "github.com/google/uuid" @@ -198,14 +199,25 @@ func (h *HostUsecase) InstallScript(ctx context.Context, token *domain.InstallRe return "", errcode.ErrInvalidInstallToken } - tmp, err := template.New("install").Parse(string(templates.InstallTmpl)) + tplName := "install" + tplContent := templates.InstallTmpl + if h.cfg.HostInstaller.Mode == "offline" { + tplName = "install_offline" + tplContent = templates.InstallOfflineTmpl + } + + tmp, err := template.New(tplName).Parse(string(tplContent)) if err != nil { return "", fmt.Errorf("failed to parse template %s", err) } buf := bytes.NewBuffer([]byte("")) param := map[string]any{ - "token": token.Token, - "grpc_url": h.cfg.TaskFlow.GrpcURL, + "token": token.Token, + "grpc_url": h.cfg.TaskFlow.GrpcURL, + "base_url": h.cfg.Server.BaseURL, + "installer_url": h.installerURL(), + "docker_bundle_path": h.installerBundlePath("docker.tgz"), + "host_bundle_path": h.hostBundlePath(), } if err := tmp.Execute(buf, param); err != nil { return "", fmt.Errorf("failed to execute template %s", err) @@ -213,6 +225,30 @@ func (h *HostUsecase) InstallScript(ctx context.Context, token *domain.InstallRe return buf.String(), nil } +func (h *HostUsecase) installerURL() string { + if h.cfg.Server.BaseURL == "" { + return "" + } + baseurl, err := url.Parse(h.cfg.Server.BaseURL) + if err != nil { + return "" + } + baseurl = baseurl.JoinPath(h.cfg.StaticFiles.RoutePrefix, "installer") + return strings.TrimRight(baseurl.String(), "/") + "/{{.arch}}/installer" +} + +func (h *HostUsecase) hostBundlePath() string { + bundlePath := h.cfg.HostInstaller.BundlePath + if bundlePath == "" { + bundlePath = "installer/{{.arch}}/host.tgz" + } + return "/" + strings.Trim(h.cfg.StaticFiles.RoutePrefix, "/") + "/" + strings.TrimLeft(bundlePath, "/") +} + +func (h *HostUsecase) installerBundlePath(name string) string { + return "/" + strings.Trim(h.cfg.StaticFiles.RoutePrefix, "/") + "/installer/{{.arch}}/" + name +} + // List implements domain.HostUsecase. func (h *HostUsecase) List(ctx context.Context, uid uuid.UUID) (*domain.HostListResp, error) { user, err := h.userRepo.Get(ctx, uid) diff --git a/backend/biz/host/usecase/host_test.go b/backend/biz/host/usecase/host_test.go index bf16848c..f4bad352 100644 --- a/backend/biz/host/usecase/host_test.go +++ b/backend/biz/host/usecase/host_test.go @@ -4,12 +4,16 @@ import ( "context" "io" "log/slog" + "strings" "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/google/uuid" _ "github.com/mattn/go-sqlite3" + "github.com/redis/go-redis/v9" + "github.com/chaitin/MonkeyCode/backend/config" "github.com/chaitin/MonkeyCode/backend/consts" "github.com/chaitin/MonkeyCode/backend/db" "github.com/chaitin/MonkeyCode/backend/db/enttest" @@ -17,6 +21,98 @@ import ( "github.com/chaitin/MonkeyCode/backend/pkg/taskflow" ) +func TestInstallScriptDefaultsToOnlineInstaller(t *testing.T) { + t.Parallel() + + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + token := "install-token" + if err := rdb.Set(context.Background(), "host:token:"+token, "1", time.Minute).Err(); err != nil { + t.Fatal(err) + } + u := &HostUsecase{ + cfg: &config.Config{ + TaskFlow: config.TaskFlow{GrpcURL: "121.41.208.82:50443"}, + }, + redis: rdb, + } + + script, err := u.InstallScript(context.Background(), &domain.InstallReq{Token: token}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(script, "release.baizhi.cloud/monkeycode/runner/$ARCH/installer") { + t.Fatalf("script missing online installer: %s", script) + } + if !strings.Contains(script, "--env GRPC_URL=121.41.208.82:50443") { + t.Fatalf("script missing grpc url: %s", script) + } + if strings.Contains(script, "install_docker_from_bundle") { + t.Fatalf("online script should not include offline installer: %s", script) + } +} + +func TestInstallScriptUsesOfflineBundle(t *testing.T) { + t.Parallel() + + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + + token := "install-token" + if err := rdb.Set(context.Background(), "host:token:"+token, "1", time.Minute).Err(); err != nil { + t.Fatal(err) + } + u := &HostUsecase{ + cfg: &config.Config{ + Server: struct { + Addr string `mapstructure:"addr"` + BaseURL string `mapstructure:"base_url"` + }{BaseURL: "http://monkeycode.local"}, + TaskFlow: config.TaskFlow{GrpcURL: "121.41.208.82:50443"}, + StaticFiles: config.StaticFilesConfig{ + RoutePrefix: "/static", + }, + HostInstaller: config.HostInstaller{ + Mode: "offline", + BundlePath: "installer/{{.arch}}/host.tgz", + }, + }, + redis: rdb, + } + + script, err := u.InstallScript(context.Background(), &domain.InstallReq{Token: token}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(script, "GRPC_URL=\"121.41.208.82:50443\"") { + t.Fatalf("script missing grpc url: %s", script) + } + if !strings.Contains(script, "INSTALLER_URL=\"http://monkeycode.local/static/installer/{{.arch}}/installer\"") { + t.Fatalf("script missing installer url: %s", script) + } + if !strings.Contains(script, "BASE_URL=\"http://monkeycode.local\"") || !strings.Contains(script, "MCAI_BASE_URL=\"$BASE_URL\"") { + t.Fatalf("script missing base url: %s", script) + } + if !strings.Contains(script, "HOST_BUNDLE_PATH=\"/static/installer/{{.arch}}/host.tgz\"") || !strings.Contains(script, "HOST_BUNDLE_PATH=${HOST_BUNDLE_PATH//\\{\\{.arch\\}\\}/$ARCH}") || !strings.Contains(script, "MCAI_HOST_BUNDLE_PATH=\"$HOST_BUNDLE_PATH\"") { + t.Fatalf("script missing host bundle path: %s", script) + } + if !strings.Contains(script, "DOCKER_BUNDLE_PATH=\"/static/installer/{{.arch}}/docker.tgz\"") || !strings.Contains(script, "DOCKER_BUNDLE_PATH=${DOCKER_BUNDLE_PATH//\\{\\{.arch\\}\\}/$ARCH}") || !strings.Contains(script, "MCAI_DOCKER_BUNDLE_PATH=\"$DOCKER_BUNDLE_PATH\"") { + t.Fatalf("script missing docker bundle path: %s", script) + } + if !strings.Contains(script, "TOKEN=\"install-token\"") || !strings.Contains(script, "MCAI_HOST_TOKEN=\"$TOKEN\"") { + t.Fatalf("script missing host token: %s", script) + } + if strings.Contains(script, "docker load") || strings.Contains(script, "docker compose") { + t.Fatalf("bootstrap script should not install host directly: %s", script) + } + if strings.Contains(script, "release.baizhi.cloud") { + t.Fatalf("script should not download public installer: %s", script) + } +} + func TestHostUsecase_markRecycledTasksFinished(t *testing.T) { t.Parallel() diff --git a/backend/biz/host/usecase/publichost_test.go b/backend/biz/host/usecase/publichost_test.go index 21bb07be..9286e7d7 100644 --- a/backend/biz/host/usecase/publichost_test.go +++ b/backend/biz/host/usecase/publichost_test.go @@ -64,12 +64,13 @@ func TestPickHostSelectsHostByRandomOffset(t *testing.T) { } } -func TestPickHostTreatsNonPositiveWeightsAsOne(t *testing.T) { +func TestPickHostIgnoresNonPositiveWeights(t *testing.T) { u := &PublicHostUsecase{ repo: &publicHostRepoStub{ hosts: []*db.Host{ {ID: "host-a", Hostname: "a", Weight: 0}, {ID: "host-b", Hostname: "b", Weight: -2}, + {ID: "host-c", Hostname: "c", Weight: 1}, }, }, taskflow: &taskflowClientStub{ @@ -77,6 +78,7 @@ func TestPickHostTreatsNonPositiveWeightsAsOne(t *testing.T) { onlineMap: map[string]bool{ "host-a": true, "host-b": true, + "host-c": true, }, }, }, @@ -84,10 +86,10 @@ func TestPickHostTreatsNonPositiveWeightsAsOne(t *testing.T) { prevRandUint64n := randUint64n randUint64n = func(n uint64) (uint64, error) { - if n != 2 { - t.Fatalf("rand limit = %d, want 2", n) + if n != 1 { + t.Fatalf("rand limit = %d, want 1", n) } - return 1, nil + return 0, nil } t.Cleanup(func() { randUint64n = prevRandUint64n @@ -97,8 +99,8 @@ func TestPickHostTreatsNonPositiveWeightsAsOne(t *testing.T) { if err != nil { t.Fatalf("PickHost() error = %v", err) } - if host.ID != "host-b" { - t.Fatalf("PickHost() = %q, want %q", host.ID, "host-b") + if host.ID != "host-c" { + t.Fatalf("PickHost() = %q, want %q", host.ID, "host-c") } } diff --git a/backend/biz/llmproxy/proxy.go b/backend/biz/llmproxy/proxy.go new file mode 100644 index 00000000..f9c124b6 --- /dev/null +++ b/backend/biz/llmproxy/proxy.go @@ -0,0 +1,207 @@ +package llmproxy + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "path" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/db" + "github.com/chaitin/MonkeyCode/backend/db/modelapikey" +) + +const upstreamFailureMessage = "连接上游模型失败，请检查模型配置，或重试" + +var allowPaths = map[string]string{ + "/v1/chat/completions": "/chat/completions", + "/v1/responses": "/responses", + "/v1/messages": "/messages", +} + +type contextKey struct{} + +type modelContext struct { + modelName string + baseURL string + apiKey string +} + +type proxyContext struct { + model *modelContext + upstreamPath string +} + +type Proxy struct { + db *db.Client + logger *slog.Logger + transport *http.Transport + proxy *httputil.ReverseProxy +} + +func NewProxy(db *db.Client, logger *slog.Logger) *Proxy { + if logger == nil { + logger = slog.Default() + } + p := &Proxy{ + db: db, + logger: logger.With("module", "llmproxy"), + transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + MaxConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 5 * time.Second, + ResponseHeaderTimeout: 300 * time.Second, + }, + } + p.proxy = &httputil.ReverseProxy{ + Transport: p.transport, + Rewrite: p.rewrite, + ErrorHandler: p.errorHandler, + FlushInterval: 100 * time.Millisecond, + } + return p +} + +func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + upstreamPath, ok := allowPaths[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + token, ok := extractToken(r) + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Bad Request", http.StatusBadRequest) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + r.ContentLength = int64(len(body)) + + reqModel, err := readRequestModel(body) + if err != nil { + http.Error(w, "Bad Request", http.StatusBadRequest) + return + } + m, err := p.resolveModel(r.Context(), token) + if err != nil { + p.logger.WarnContext(r.Context(), "resolve runtime model failed", "error", err) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if reqModel != "" && reqModel != m.modelName { + p.logger.WarnContext(r.Context(), "model mismatch", "request_model", reqModel, "expected_model", m.modelName) + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + + ctx := context.WithValue(r.Context(), contextKey{}, &proxyContext{ + model: m, + upstreamPath: upstreamPath, + }) + p.proxy.ServeHTTP(w, r.WithContext(ctx)) +} + +func (p *Proxy) resolveModel(ctx context.Context, token string) (*modelContext, error) { + keyID, err := uuid.Parse(token) + query := p.db.ModelApiKey.Query(). + WithModel(). + Where(modelapikey.APIKey(token)) + if err == nil { + query = p.db.ModelApiKey.Query(). + WithModel(). + Where(modelapikey.Or(modelapikey.ID(keyID), modelapikey.APIKey(token))) + } + key, err := query.Only(ctx) + if err != nil { + return nil, err + } + if key.Edges.Model == nil { + return nil, errors.New("model not found") + } + return &modelContext{ + modelName: key.Edges.Model.Model, + baseURL: key.Edges.Model.BaseURL, + apiKey: key.Edges.Model.APIKey, + }, nil +} + +func (p *Proxy) rewrite(r *httputil.ProxyRequest) { + ctx, ok := r.In.Context().Value(contextKey{}).(*proxyContext) + if !ok || ctx == nil || ctx.model == nil { + p.logger.WarnContext(r.In.Context(), "missing model context") + return + } + m := ctx.model + baseURL, err := url.Parse(m.baseURL) + if err != nil { + p.logger.ErrorContext(r.In.Context(), "parse model base url failed", "base_url", m.baseURL, "error", err) + return + } + r.Out.URL.Scheme = baseURL.Scheme + r.Out.URL.Host = baseURL.Host + r.Out.URL.Path = joinURLPath(baseURL.Path, ctx.upstreamPath) + r.Out.URL.RawQuery = r.In.URL.RawQuery + r.Out.Host = baseURL.Host + r.Out.Header.Set("Authorization", "Bearer "+m.apiKey) + r.Out.Header.Set("X-Api-Key", m.apiKey) + r.SetXForwarded() +} + +func (p *Proxy) errorHandler(w http.ResponseWriter, r *http.Request, err error) { + p.logger.ErrorContext(r.Context(), "proxy upstream failed", "path", r.URL.Path, "error", err) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(upstreamFailureMessage)) +} + +func extractToken(req *http.Request) (string, bool) { + token := strings.TrimSpace(req.Header.Get("X-Api-Key")) + if token != "" { + return token, true + } + token, ok := strings.CutPrefix(req.Header.Get("Authorization"), "Bearer ") + if !ok { + return "", false + } + token = strings.TrimSpace(token) + return token, token != "" +} + +func readRequestModel(body []byte) (string, error) { + var payload struct { + Model string `json:"model"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return "", fmt.Errorf("parse llm request: %w", err) + } + return payload.Model, nil +} + +func joinURLPath(basePath, requestPath string) string { + if basePath == "" || basePath == "/" { + return requestPath + } + return path.Join(basePath, requestPath) +} diff --git a/backend/biz/llmproxy/proxy_test.go b/backend/biz/llmproxy/proxy_test.go new file mode 100644 index 00000000..4ded27f8 --- /dev/null +++ b/backend/biz/llmproxy/proxy_test.go @@ -0,0 +1,166 @@ +package llmproxy + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/uuid" + _ "github.com/mattn/go-sqlite3" + + "github.com/chaitin/MonkeyCode/backend/consts" + "github.com/chaitin/MonkeyCode/backend/db" + "github.com/chaitin/MonkeyCode/backend/db/enttest" +) + +func newProxyTestDB(t *testing.T) *db.Client { + t.Helper() + client := enttest.Open(t, "sqlite3", "file:llmproxy-test?mode=memory&cache=shared&_fk=1") + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func seedProxyModel(t *testing.T, client *db.Client, upstreamURL string) string { + t.Helper() + ctx := context.Background() + userID := uuid.New() + modelID := uuid.New() + key := "runtime-" + uuid.NewString() + + if _, err := client.User.Create(). + SetID(userID). + SetName("user"). + SetRole(consts.UserRoleIndividual). + SetStatus(consts.UserStatusActive). + Save(ctx); err != nil { + t.Fatal(err) + } + if _, err := client.Model.Create(). + SetID(modelID). + SetUserID(userID). + SetProvider("OpenAI"). + SetAPIKey("real-model-key"). + SetBaseURL(upstreamURL). + SetModel("gpt-4o"). + Save(ctx); err != nil { + t.Fatal(err) + } + if _, err := client.ModelApiKey.Create(). + SetID(uuid.New()). + SetUserID(userID). + SetModelID(modelID). + SetAPIKey(key). + Save(ctx); err != nil { + t.Fatal(err) + } + return key +} + +func TestProxyForwardsRuntimeKeyToUpstreamModel(t *testing.T) { + var gotPath string + var gotAuth string + var gotBody string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + gotBody = string(body) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"id":"chatcmpl_test","choices":[{"message":{"content":"ok"}}]}`)) + })) + t.Cleanup(upstream.Close) + + client := newProxyTestDB(t) + runtimeKey := seedProxyModel(t, client, upstream.URL+"/v1") + proxy := NewProxy(client, slog.New(slog.NewTextHandler(io.Discard, nil))) + + body := `{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}]}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+runtimeKey) + rec := httptest.NewRecorder() + + proxy.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if gotPath != "/v1/chat/completions" { + t.Fatalf("upstream path = %q", gotPath) + } + if gotAuth != "Bearer real-model-key" { + t.Fatalf("upstream auth = %q", gotAuth) + } + if gotBody != body { + t.Fatalf("upstream body = %q", gotBody) + } +} + +func TestProxyRejectsMissingRuntimeKey(t *testing.T) { + client := newProxyTestDB(t) + proxy := NewProxy(client, slog.New(slog.NewTextHandler(io.Discard, nil))) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"gpt-4o"}`)) + rec := httptest.NewRecorder() + + proxy.ServeHTTP(rec, req) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } +} + +func TestProxyRejectsModelMismatch(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("upstream should not be called") + })) + t.Cleanup(upstream.Close) + + client := newProxyTestDB(t) + runtimeKey := seedProxyModel(t, client, upstream.URL) + proxy := NewProxy(client, slog.New(slog.NewTextHandler(io.Discard, nil))) + + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"other-model"}`)) + req.Header.Set("Authorization", "Bearer "+runtimeKey) + rec := httptest.NewRecorder() + + proxy.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } +} + +func TestProxyAppendsEndpointToVersionedBaseURL(t *testing.T) { + var gotPath string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"id":"resp_test"}`)) + })) + t.Cleanup(upstream.Close) + + client := newProxyTestDB(t) + runtimeKey := seedProxyModel(t, client, upstream.URL+"/v1") + proxy := NewProxy(client, slog.New(slog.NewTextHandler(io.Discard, nil))) + + req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"model":"gpt-4o","input":"hi"}`)) + req.Header.Set("X-Api-Key", runtimeKey) + rec := httptest.NewRecorder() + + proxy.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if gotPath != "/v1/responses" { + t.Fatalf("upstream path = %q, want /v1/responses", gotPath) + } +} diff --git a/backend/biz/llmproxy/register.go b/backend/biz/llmproxy/register.go new file mode 100644 index 00000000..db458f5c --- /dev/null +++ b/backend/biz/llmproxy/register.go @@ -0,0 +1,40 @@ +package llmproxy + +import ( + "log/slog" + + "github.com/GoYoko/web" + "github.com/samber/do" + + "github.com/chaitin/MonkeyCode/backend/db" +) + +type Handler struct { + proxy *Proxy +} + +func ProvideLLMProxy(i *do.Injector) { + do.Provide(i, NewHandler) +} + +func InvokeLLMProxy(i *do.Injector) { + do.MustInvoke[*Handler](i) +} + +func NewHandler(i *do.Injector) (*Handler, error) { + w := do.MustInvoke[*web.Web](i) + client := do.MustInvoke[*db.Client](i) + logger := do.MustInvoke[*slog.Logger](i) + + h := &Handler{proxy: NewProxy(client, logger)} + g := w.Group("/v1") + g.POST("/chat/completions", web.BaseHandler(h.ServeHTTP)) + g.POST("/responses", web.BaseHandler(h.ServeHTTP)) + g.POST("/messages", web.BaseHandler(h.ServeHTTP)) + return h, nil +} + +func (h *Handler) ServeHTTP(c *web.Context) error { + h.proxy.ServeHTTP(c.Response(), c.Request()) + return nil +} diff --git a/backend/biz/llmproxy/register_test.go b/backend/biz/llmproxy/register_test.go new file mode 100644 index 00000000..5fa16fb5 --- /dev/null +++ b/backend/biz/llmproxy/register_test.go @@ -0,0 +1,43 @@ +package llmproxy + +import ( + "io" + "log/slog" + "testing" + + "github.com/GoYoko/web" + "github.com/samber/do" + + "github.com/chaitin/MonkeyCode/backend/db" +) + +func TestNewHandlerRegistersProxyRoutes(t *testing.T) { + injector := do.New() + w := web.New() + do.ProvideValue(injector, w) + do.ProvideValue(injector, newProxyTestDB(t)) + do.ProvideValue(injector, slog.New(slog.NewTextHandler(io.Discard, nil))) + + if _, err := NewHandler(injector); err != nil { + t.Fatal(err) + } + + want := map[string]bool{ + "POST /v1/chat/completions": false, + "POST /v1/responses": false, + "POST /v1/messages": false, + } + for _, route := range w.Routes() { + key := route.Method + " " + route.Path + if _, ok := want[key]; ok { + want[key] = true + } + } + for route, found := range want { + if !found { + t.Fatalf("route %s is not registered", route) + } + } + + _ = do.MustInvoke[*db.Client](injector) +} diff --git a/backend/biz/register.go b/backend/biz/register.go index 8e227ac7..7b76d12f 100644 --- a/backend/biz/register.go +++ b/backend/biz/register.go @@ -6,12 +6,15 @@ import ( "github.com/chaitin/MonkeyCode/backend/biz/file" "github.com/chaitin/MonkeyCode/backend/biz/git" "github.com/chaitin/MonkeyCode/backend/biz/host" + "github.com/chaitin/MonkeyCode/backend/biz/llmproxy" "github.com/chaitin/MonkeyCode/backend/biz/notify" "github.com/chaitin/MonkeyCode/backend/biz/project" "github.com/chaitin/MonkeyCode/backend/biz/public" "github.com/chaitin/MonkeyCode/backend/biz/setting" + "github.com/chaitin/MonkeyCode/backend/biz/static" "github.com/chaitin/MonkeyCode/backend/biz/task" "github.com/chaitin/MonkeyCode/backend/biz/team" + "github.com/chaitin/MonkeyCode/backend/biz/uploader" "github.com/chaitin/MonkeyCode/backend/biz/user" "github.com/chaitin/MonkeyCode/backend/biz/vmidle" ) @@ -19,7 +22,6 @@ import ( // RegisterAll 注册所有 biz 模块 // 分两阶段：先 Provide（懒注册），再 Invoke（解析依赖），避免模块间循环依赖 func RegisterAll(i *do.Injector) error { - // 阶段一：所有模块注册服务工厂（do.Provide，不触发依赖解析） notify.ProvideNotify(i) public.ProvidePublic(i) user.ProvideUser(i) @@ -31,8 +33,10 @@ func RegisterAll(i *do.Injector) error { project.ProvideProject(i) file.ProvideFile(i) vmidle.ProvideVMIdle(i) + return nil +} - // 阶段二：统一触发 handler 初始化（do.MustInvoke，此时所有服务已注册） +func InvokeAll(i *do.Injector) { notify.InvokeNotify(i) public.InvokePublic(i) user.InvokeUser(i) @@ -44,6 +48,17 @@ func RegisterAll(i *do.Injector) error { project.InvokeProject(i) file.InvokeFile(i) vmidle.InvokeVMIdle(i) +} - return nil +// RegisterOpenSource 注册仅在开源项目中使用的模块 +func RegisterOpenSource(i *do.Injector) { + uploader.ProvideUploader(i) + llmproxy.ProvideLLMProxy(i) + static.ProviderStatic(i) +} + +func InvokeOpenSource(i *do.Injector) { + uploader.InvokeUploader(i) + llmproxy.InvokeLLMProxy(i) + static.InvokeStatic(i) } diff --git a/backend/biz/static/handler/static.go b/backend/biz/static/handler/static.go new file mode 100644 index 00000000..e58b2b36 --- /dev/null +++ b/backend/biz/static/handler/static.go @@ -0,0 +1,21 @@ +package handler + +import ( + "github.com/GoYoko/web" + "github.com/samber/do" + + "github.com/chaitin/MonkeyCode/backend/config" +) + +type StaticHandler struct { +} + +func NewStaticHandler(i *do.Injector) (*StaticHandler, error) { + w := do.MustInvoke[*web.Web](i) + cfg := do.MustInvoke[*config.Config](i) + + s := &StaticHandler{} + + w.Echo().Static(cfg.StaticFiles.RoutePrefix, cfg.StaticFiles.Dir) + return s, nil +} diff --git a/backend/biz/static/register.go b/backend/biz/static/register.go new file mode 100644 index 00000000..0bfe85cf --- /dev/null +++ b/backend/biz/static/register.go @@ -0,0 +1,16 @@ +package static + +import ( + "github.com/samber/do" + + "github.com/chaitin/MonkeyCode/backend/biz/static/handler" +) + +func ProviderStatic(i *do.Injector) { + do.Provide(i, handler.NewStaticHandler) + handler.NewStaticHandler(i) +} + +func InvokeStatic(i *do.Injector) { + do.MustInvoke[*handler.StaticHandler](i) +} diff --git a/backend/biz/uploader/handler/http/v1/uploader.go b/backend/biz/uploader/handler/http/v1/uploader.go new file mode 100644 index 00000000..7f8479b9 --- /dev/null +++ b/backend/biz/uploader/handler/http/v1/uploader.go @@ -0,0 +1,175 @@ +package v1 + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/GoYoko/web" + "github.com/google/uuid" + "github.com/samber/do" + + "github.com/chaitin/MonkeyCode/backend/config" + "github.com/chaitin/MonkeyCode/backend/consts" + "github.com/chaitin/MonkeyCode/backend/domain" + "github.com/chaitin/MonkeyCode/backend/errcode" + "github.com/chaitin/MonkeyCode/backend/middleware" + "github.com/chaitin/MonkeyCode/backend/pkg/oss" +) + +const defaultUploadMaxSize = 50 << 20 + +var allowedExtensions = map[string]bool{ + ".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true, ".ico": true, ".bmp": true, + ".pdf": true, ".doc": true, ".docx": true, ".xls": true, ".xlsx": true, ".ppt": true, ".pptx": true, + ".txt": true, ".md": true, ".markdown": true, ".csv": true, ".json": true, ".yaml": true, ".yml": true, + ".zip": true, ".tar": true, ".gz": true, ".tgz": true, ".rar": true, ".7z": true, + ".exe": true, ".dmg": true, ".deb": true, ".rpm": true, +} + +type UploaderHandler struct { + cfg *config.Config + logger *slog.Logger + client *oss.Client +} + +func NewUploaderHandler(i *do.Injector) (*UploaderHandler, error) { + cfg := do.MustInvoke[*config.Config](i) + if !cfg.ObjectStorage.Enabled { + return &UploaderHandler{cfg: cfg}, nil + } + w := do.MustInvoke[*web.Web](i) + auth := do.MustInvoke[*middleware.AuthMiddleware](i) + targetActive := do.MustInvoke[*middleware.TargetActiveMiddleware](i) + logger := do.MustInvoke[*slog.Logger](i).With("module", "handler.uploader") + opt := oss.S3Option{ForcePathStyle: cfg.ObjectStorage.ForcePathStyle, InitBucket: cfg.ObjectStorage.InitBucket} + client, err := oss.NewS3Compatible(context.Background(), cfg.ObjectStorage, opt) + if err != nil { + return nil, err + } + h := &UploaderHandler{cfg: cfg, logger: logger, client: client} + g := w.Group("/api/v1/uploader") + g.Use(auth.Auth(), targetActive.TargetActive()) + g.POST("", web.BindHandler(h.Upload)) + g.POST("/presign", web.BindHandler(h.Presign)) + return h, nil +} + +func (h *UploaderHandler) Upload(c *web.Context, req domain.UploadReq) error { + if h == nil || h.client == nil { + return errcode.ErrBadRequest.Wrap(fmt.Errorf("object storage is disabled")) + } + user := middleware.GetUser(c) + if user == nil { + return errcode.ErrUnauthorized + } + if req.File == nil { + return errcode.ErrBadRequest.Wrap(fmt.Errorf("file is required")) + } + maxSize := h.cfg.ObjectStorage.MaxSize + if maxSize <= 0 { + maxSize = defaultUploadMaxSize + } + if req.File.Size > maxSize { + return errcode.ErrBadRequest.Wrap(fmt.Errorf("file exceeds limit")) + } + file, err := req.File.Open() + if err != nil { + return err + } + defer file.Close() + fileData, err := io.ReadAll(io.LimitReader(file, maxSize+1)) + if err != nil { + return err + } + if int64(len(fileData)) > maxSize { + return errcode.ErrBadRequest.Wrap(fmt.Errorf("file exceeds limit")) + } + ext := strings.ToLower(filepath.Ext(req.File.Filename)) + if ext != "" && !allowedExtension(ext) { + return errcode.ErrBadRequest.Wrap(fmt.Errorf("unsupported file extension")) + } + filename := fmt.Sprintf("%s_%s%s", user.ID.String(), fileMD5(fileData), ext) + prefix, err := h.uploadPrefix(req.Usage) + if err != nil { + return err + } + client := h.requestClient(c.Request()) + if err := client.PutFile(c.Request().Context(), prefix, filename, bytes.NewReader(fileData)); err != nil { + h.logger.With("error", err).ErrorContext(c.Request().Context(), "upload object failed") + return err + } + return c.Success(client.GetURL(prefix, filename)) +} + +func (h *UploaderHandler) Presign(c *web.Context, req domain.PresignReq) error { + if h == nil || h.client == nil { + return errcode.ErrBadRequest.Wrap(fmt.Errorf("object storage is disabled")) + } + user := middleware.GetUser(c) + if user == nil { + return errcode.ErrUnauthorized + } + filename, err := presignFilename(user.ID, req.Filename) + if err != nil { + return err + } + presign, err := h.requestClient(c.Request()).Presign(c.Request().Context(), h.cfg.ObjectStorage.TempPrefix, filename, parsePresignExpires(h.cfg.ObjectStorage.PresignExpires)) + if err != nil { + h.logger.With("error", err).ErrorContext(c.Request().Context(), "presign object failed") + return err + } + return c.Success(domain.PresignResp{UploadURL: presign.UploadURL, AccessURL: presign.AccessURL}) +} + +func (h *UploaderHandler) requestClient(r *http.Request) *oss.Client { + return h.client.WithAccessEndpoint(h.cfg.ObjectStorage.AccessEndpoint) +} + +func (h *UploaderHandler) uploadPrefix(usage consts.UploadUsage) (string, error) { + switch usage { + case consts.UploadUsageAvatar: + return h.cfg.ObjectStorage.AvatarPrefix, nil + case consts.UploadUsageSpec: + return h.cfg.ObjectStorage.SpecPrefix, nil + case consts.UploadUsageRepo: + return h.cfg.ObjectStorage.RepoPrefix, nil + default: + return "", errcode.ErrBadRequest.Wrap(fmt.Errorf("unsupported upload usage")) + } +} + +func allowedExtension(ext string) bool { + return allowedExtensions[strings.ToLower(ext)] +} + +func presignFilename(userID uuid.UUID, original string) (string, error) { + ext := strings.ToLower(filepath.Ext(original)) + hash := md5.Sum([]byte(original)) + return fmt.Sprintf("%s_%s%s", userID.String(), hex.EncodeToString(hash[:]), ext), nil +} + +func fileMD5(fileb []byte) string { + hash := md5.New() + hash.Write(fileb) + return hex.EncodeToString(hash.Sum(nil)) +} + +func parsePresignExpires(raw string) time.Duration { + expires, err := time.ParseDuration(strings.TrimSpace(raw)) + if err != nil || expires <= 0 { + return 7 * 24 * time.Hour + } + if expires > 7*24*time.Hour { + return 7 * 24 * time.Hour + } + return expires +} diff --git a/backend/biz/uploader/handler/http/v1/uploader_test.go b/backend/biz/uploader/handler/http/v1/uploader_test.go new file mode 100644 index 00000000..cda8f682 --- /dev/null +++ b/backend/biz/uploader/handler/http/v1/uploader_test.go @@ -0,0 +1,85 @@ +package v1 + +import ( + "context" + "net/http/httptest" + "testing" + "time" + + "github.com/chaitin/MonkeyCode/backend/config" + "github.com/chaitin/MonkeyCode/backend/consts" + "github.com/chaitin/MonkeyCode/backend/pkg/oss" + "github.com/google/uuid" +) + +func TestPresignFilenameKeepsLowercaseExtension(t *testing.T) { + userID := uuid.MustParse("11111111-1111-1111-1111-111111111111") + got, err := presignFilename(userID, "archive.ZIP") + if err != nil { + t.Fatal(err) + } + want := "11111111-1111-1111-1111-111111111111_670070ac98fc89f453cdd612492fc0df.zip" + if got != want { + t.Fatalf("filename = %q, want %q", got, want) + } +} + +func TestAllowedExtensionAllowsMarkdown(t *testing.T) { + if !allowedExtension(".MD") { + t.Fatal("expected .MD allowed") + } +} + +func TestAllowedExtensionRejectsScript(t *testing.T) { + if allowedExtension(".sh") { + t.Fatal("expected .sh rejected") + } +} + +func TestParsePresignExpiresDefaultsToSevenDays(t *testing.T) { + got := parsePresignExpires("") + if got != 7*24*time.Hour { + t.Fatalf("expires = %s", got) + } +} + +func TestParsePresignExpiresClampsToSevenDays(t *testing.T) { + got := parsePresignExpires("240h") + if got != 7*24*time.Hour { + t.Fatalf("expires = %s", got) + } +} + +func TestUploadPrefixSelectsRepoPrefix(t *testing.T) { + h := &UploaderHandler{cfg: &config.Config{}} + h.cfg.ObjectStorage.RepoPrefix = "repo" + got, err := h.uploadPrefix(consts.UploadUsageRepo) + if err != nil { + t.Fatal(err) + } + if got != "repo" { + t.Fatalf("prefix = %q", got) + } +} + +func TestRequestObjectStorageClientUsesConfiguredAccessEndpoint(t *testing.T) { + h := &UploaderHandler{cfg: &config.Config{}} + h.cfg.ObjectStorage.AccessEndpoint = "https://monkeycode.example.com/oss" + h.cfg.ObjectStorage.Bucket = "monkeycode-private" + client, err := oss.NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: "http://internal:9000", + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "monkeycode-private", + }, oss.S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + h.client = client + req := httptest.NewRequest("POST", "http://internal:8888/api/v1/uploader/presign", nil) + + got := h.requestClient(req).GetURL("tmp", "a.txt") + if got != "https://monkeycode.example.com/oss/monkeycode-private/tmp/a.txt" { + t.Fatalf("url = %q", got) + } +} diff --git a/backend/biz/uploader/register.go b/backend/biz/uploader/register.go new file mode 100644 index 00000000..5c7539ba --- /dev/null +++ b/backend/biz/uploader/register.go @@ -0,0 +1,20 @@ +package uploader + +import ( + "github.com/samber/do" + + v1 "github.com/chaitin/MonkeyCode/backend/biz/uploader/handler/http/v1" + "github.com/chaitin/MonkeyCode/backend/config" +) + +func ProvideUploader(i *do.Injector) { + do.Provide(i, v1.NewUploaderHandler) +} + +func InvokeUploader(i *do.Injector) { + cfg := do.MustInvoke[*config.Config](i) + if !cfg.ObjectStorage.Enabled { + return + } + do.MustInvoke[*v1.UploaderHandler](i) +} diff --git a/backend/biz/uploader/register_test.go b/backend/biz/uploader/register_test.go new file mode 100644 index 00000000..2c62b499 --- /dev/null +++ b/backend/biz/uploader/register_test.go @@ -0,0 +1,12 @@ +package uploader + +import ( + "testing" + + "github.com/samber/do" +) + +func TestProvideUploader(t *testing.T) { + i := do.New() + ProvideUploader(i) +} diff --git a/backend/biz/user/handler/v1/auth.go b/backend/biz/user/handler/v1/auth.go index d1a83fa4..bfbb20c2 100644 --- a/backend/biz/user/handler/v1/auth.go +++ b/backend/biz/user/handler/v1/auth.go @@ -57,6 +57,7 @@ func NewAuthHandler(i *do.Injector) (*AuthHandler, error) { // 密码登录 v1.POST("/password-login", web.BindHandler(h.PasswordLogin), targetActive.TargetActive()) + v1.PUT("", web.BindHandler(h.Update), auth.Auth(), targetActive.TargetActive()) v1.PUT("/passwords/change", web.BindHandler(h.ChangePassword), auth.Check(), targetActive.TargetActive()) v1.GET("/status", web.BaseHandler(h.Status), auth.Check(), targetActive.TargetActive()) v1.POST("/logout", web.BaseHandler(h.Logout), auth.Auth(), targetActive.TargetActive()) @@ -102,6 +103,37 @@ func (h *AuthHandler) PasswordLogin(c *web.Context, req domain.TeamLoginReq) err return c.Success(user) } +// Update 更新用户信息 +// +// @Summary 更新用户信息 +// @Description 更新用户昵称和头像 +// @Tags 【用户】用户 +// @Accept multipart/form-data +// @Produce json +// @Security MonkeyCodeAIAuth +// @Param name formData string false "昵称" +// @Param avatar_url formData string false "OSS 头像地址" +// @Success 200 {object} web.Resp{data=domain.UpdateUserResp} +// @Router /api/v1/users [put] +func (h *AuthHandler) Update(c *web.Context, req domain.UpdateUserReq) error { + user := middleware.GetUser(c) + if user == nil { + return errcode.ErrUnauthorized + } + + updated, err := h.usecase.Update(c.Request().Context(), user.ID, req.AvatarURL, req) + if err != nil { + h.logger.ErrorContext(c.Request().Context(), "update user failed", "error", err, "user_id", user.ID) + return err + } + + return c.Success(domain.UpdateUserResp{ + User: updated, + Message: "success", + Success: true, + }) +} + // ChangePassword 修改密码接口 // // @Summary 修改密码 diff --git a/backend/biz/user/handler/v1/auth_update_test.go b/backend/biz/user/handler/v1/auth_update_test.go new file mode 100644 index 00000000..c0edfc63 --- /dev/null +++ b/backend/biz/user/handler/v1/auth_update_test.go @@ -0,0 +1,122 @@ +package v1 + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/GoYoko/web" + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "github.com/chaitin/MonkeyCode/backend/domain" + "github.com/chaitin/MonkeyCode/backend/middleware" +) + +func TestUpdateUserRoute(t *testing.T) { + userID := uuid.MustParse("11111111-1111-1111-1111-111111111111") + w := web.New() + h := &AuthHandler{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + usecase: &updateUserUsecaseStub{userID: userID}, + } + g := w.Group("/api/v1/users") + g.PUT("", web.BindHandler(h.Update), setUserMiddleware(&domain.User{ID: userID})) + + body := &strings.Builder{} + mw := multipart.NewWriter(body) + if err := mw.WriteField("name", "新昵称"); err != nil { + t.Fatal(err) + } + if err := mw.WriteField("avatar_url", "https://example.com/avatar.png"); err != nil { + t.Fatal(err) + } + if err := mw.Close(); err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPut, "/api/v1/users", strings.NewReader(body.String())) + req.Header.Set(echo.HeaderContentType, mw.FormDataContentType()) + rec := httptest.NewRecorder() + + w.Echo().ServeHTTP(rec, req) + + if rec.Code == http.StatusNotFound { + t.Fatal("PUT /api/v1/users route is not registered") + } + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + var resp struct { + Code int `json:"code"` + Data domain.UpdateUserResp `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Code != 0 { + t.Fatalf("code = %d", resp.Code) + } + if resp.Data.User == nil || resp.Data.User.Name != "新昵称" || resp.Data.User.AvatarURL != "https://example.com/avatar.png" { + t.Fatalf("data = %#v", resp.Data) + } +} + +func setUserMiddleware(user *domain.User) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + middleware.SetUser(c, user) + return next(c) + } + } +} + +type updateUserUsecaseStub struct { + domain.UserUsecase + userID uuid.UUID +} + +func (s *updateUserUsecaseStub) Update(ctx context.Context, uid uuid.UUID, avatarURL string, req domain.UpdateUserReq) (*domain.User, error) { + return &domain.User{ + ID: uid, + Name: req.Name, + AvatarURL: avatarURL, + }, nil +} + +func (s *updateUserUsecaseStub) Get(ctx context.Context, uid uuid.UUID) (*domain.User, error) { + return &domain.User{ID: uid}, nil +} + +func (s *updateUserUsecaseStub) GetUserWithTeams(ctx context.Context, userID uuid.UUID) (*domain.TeamUserInfo, error) { + return &domain.TeamUserInfo{}, nil +} + +func (s *updateUserUsecaseStub) PasswordLogin(ctx context.Context, req *domain.TeamLoginReq) (*domain.User, error) { + return nil, nil +} + +func (s *updateUserUsecaseStub) ChangePassword(ctx context.Context, userID uuid.UUID, req *domain.ChangePasswordReq, isReset bool) error { + return nil +} + +func (s *updateUserUsecaseStub) SendResetPasswordEmail(ctx context.Context, req *domain.ResetUserPasswordEmailReq) error { + return nil +} + +func (s *updateUserUsecaseStub) GetUserByEmail(ctx context.Context, emails []string) ([]*domain.User, error) { + return nil, nil +} + +func (s *updateUserUsecaseStub) SendBindEmailVerification(ctx context.Context, userID uuid.UUID, req *domain.SendBindEmailVerificationReq) error { + return nil +} + +func (s *updateUserUsecaseStub) VerifyBindEmail(ctx context.Context, token string) error { + return nil +} diff --git a/backend/bridge.go b/backend/bridge.go index b921c0f7..add075b2 100644 --- a/backend/bridge.go +++ b/backend/bridge.go @@ -121,5 +121,7 @@ func Register(e *echo.Echo, dir string, opts ...BridgeOption) error { opt(injector) } - return biz.RegisterAll(injector) + biz.RegisterAll(injector) + biz.InvokeAll(injector) + return nil } diff --git a/backend/build/nginx.conf b/backend/build/nginx.conf index 28476b4b..c2964ee8 100644 --- a/backend/build/nginx.conf +++ b/backend/build/nginx.conf @@ -5,29 +5,34 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; - + # 日志格式 log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - + # 基本设置 sendfile on; tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; - + # Gzip 压缩 gzip on; gzip_vary on; gzip_min_length 1024; gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json; + upstream rustfs { + least_conn; + server monkeycode-ai-rustfs:9000; # S3 API 服务端口 + } + server { listen 80; server_name _; - + # 处理前端路由（SPA） location / { proxy_pass http://monkeycode-ai-frontend; @@ -36,10 +41,11 @@ http { location /api/ { client_max_body_size 10m; proxy_pass http://monkeycode-ai-backend:8888; - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; # 支持 WebSocket proxy_http_version 1.1; @@ -47,16 +53,62 @@ http { proxy_set_header Connection "upgrade"; } + location /v1/ { + client_max_body_size 20m; + proxy_pass http://monkeycode-ai-backend:8888; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 24h; + proxy_connect_timeout 24h; + proxy_send_timeout 24h; + } + + location /static/ { + client_max_body_size 1024m; + proxy_pass http://monkeycode-ai-backend:8888; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + } + location /api/v1/users/files/upload { client_max_body_size 10m; proxy_pass http://monkeycode-ai-backend:8888; - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; proxy_read_timeout 10m; proxy_send_timeout 60s; } + + location /oss/ { + client_max_body_size 100m; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + + proxy_cache_convert_head off; + proxy_buffering off; + proxy_request_buffering off; + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + + proxy_pass http://rustfs/; + } } upstream grpcservers { @@ -65,10 +117,12 @@ http { } server { - listen 50051; + listen 50443 ssl; server_name _; http2 on; + ssl_certificate /etc/tls/server.crt; + ssl_certificate_key /etc/tls/server.key; underscores_in_headers on; location / { @@ -97,35 +151,4 @@ http { grpc_set_header X-Real-IP $remote_addr; } } - - upstream rustfs { - least_conn; - server monkeycode-ai-rustfs:9000; # S3 API 服务端口 - } - - server { - listen 8000; - http2 on; - server_name _; # 替换为你的 S3 API 域名 - - add_header Strict-Transport-Security "max-age=31536000"; - - # 反向代理 RustFS S3 API - location / { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Port $server_port; - proxy_set_header X-Forwarded-Host $host; - - # 关键配置：禁用 HEAD 请求转换，避免 S3 V4 签名失效 - proxy_cache_convert_head off; - proxy_connect_timeout 300; - proxy_http_version 1.1; - proxy_set_header Connection ""; - - proxy_pass http://rustfs; # 代理到 S3 API - } - } } diff --git a/backend/cmd/installer/center.go b/backend/cmd/installer/center.go new file mode 100644 index 00000000..ce81587a --- /dev/null +++ b/backend/cmd/installer/center.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/huh" + + "github.com/chaitin/MonkeyCode/backend/pkg/installer" +) + +const defaultCenterInstallDir = "/data/monkeycode-ai" + +type centerStep int + +const ( + centerStepCheck centerStep = iota + centerStepForm + centerStepConfirmDocker + centerStepRun + centerStepDone +) + +type centerModel struct { + ctx context.Context + runner installer.Runner + spinner spinner.Model + step centerStep + status installer.DockerStatus + form *huh.Form + installDir string + accessHost string + nginxPort string + teamEmail string + teamName string + teamPassword string + installDocker bool + action string + err error + result centerInstallResult +} + +type centerStatusMsg installer.DockerStatus +type centerDoneMsg struct { + err error + result centerInstallResult +} + +type centerInstallResult struct { + URL string + AdminEmail string + AdminPassword string +} + +func newCenterModel(ctx context.Context) *centerModel { + sp := spinner.New() + sp.Spinner = spinner.Dot + return ¢erModel{ + ctx: ctx, + runner: installer.CommandRunner{}, + spinner: sp, + step: centerStepCheck, + installDir: defaultCenterInstallDir, + nginxPort: "80", + teamName: "MonkeyCode", + } +} + +func (m *centerModel) Init() tea.Cmd { + return tea.Batch(m.spinner.Tick, m.checkDocker) +} + +func (m *centerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + if msg.String() == "ctrl+c" { + return m, tea.Quit + } + case centerStatusMsg: + m.status = installer.DockerStatus(msg) + if !m.status.Ready() { + m.step = centerStepConfirmDocker + m.installDocker = true + m.form = m.confirmDockerForm() + return m, m.form.Init() + } + m.step = centerStepForm + m.form = m.centerForm() + return m, m.form.Init() + case centerDoneMsg: + m.err = msg.err + m.result = msg.result + m.step = centerStepDone + if msg.err == nil { + return m, nil + } + return m, nil + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + } + if m.form != nil { + form, cmd := m.form.Update(msg) + m.form = form.(*huh.Form) + if m.form.State == huh.StateCompleted { + m.form = nil + return m.nextAfterForm() + } + if m.form.State == huh.StateAborted { + return m, tea.Quit + } + return m, cmd + } + return m, nil +} + +func (m *centerModel) View() string { + var b strings.Builder + b.WriteString(titleStyle.Render("MonkeyCode 中心端安装器")) + b.WriteString("\n\n") + switch m.step { + case centerStepCheck: + b.WriteString(m.spinner.View()) + b.WriteString(" 正在检测 Docker 环境...\n") + case centerStepForm, centerStepConfirmDocker: + b.WriteString(renderStatus(m.status)) + b.WriteString("\n") + if m.form != nil { + b.WriteString(m.form.View()) + b.WriteString("\n") + } + case centerStepRun: + b.WriteString(m.spinner.View()) + b.WriteString(" " + m.action + "...\n") + case centerStepDone: + if m.err != nil { + b.WriteString(errStyle.Render(m.action + "失败: " + m.err.Error())) + } else { + b.WriteString(okStyle.Render(m.action + "完成。")) + if m.result.URL != "" { + b.WriteString("\n\n") + b.WriteString("访问地址: " + m.result.URL + "\n") + b.WriteString("管理员账号: " + m.result.AdminEmail + "\n") + b.WriteString("管理员密码: " + m.result.AdminPassword + "\n") + } + } + b.WriteString("\n") + } + return b.String() +} + +func (m *centerModel) checkDocker() tea.Msg { + return centerStatusMsg(installer.CheckDockerStatus(m.ctx, m.runner)) +} + +func (m *centerModel) confirmDockerForm() *huh.Form { + return huh.NewForm(huh.NewGroup( + huh.NewConfirm(). + Title("安装 Docker/Compose"). + Description("当前 Docker 环境不完整，将使用离线包内 docker.tgz 安装静态二进制。"). + Affirmative("开始安装"). + Negative("取消"). + Value(&m.installDocker), + )) +} + +func (m *centerModel) centerForm() *huh.Form { + return huh.NewForm(huh.NewGroup( + huh.NewInput(). + Title("中心端安装目录"). + Placeholder(defaultCenterInstallDir). + Value(&m.installDir). + Validate(validateAbsPath), + huh.NewInput(). + Title("中心端访问地址"). + Description("请输入用户和宿主机能访问到的 IP 或域名。"). + Value(&m.accessHost). + Validate(func(value string) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("请输入中心端访问地址") + } + return nil + }), + huh.NewInput(). + Title("Nginx HTTP 端口"). + Value(&m.nginxPort). + Validate(func(value string) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("请输入 Nginx 端口") + } + return nil + }), + huh.NewInput(). + Title("管理员邮箱"). + Value(&m.teamEmail). + Validate(func(value string) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("请输入管理员邮箱") + } + return nil + }), + huh.NewInput(). + Title("团队名称"). + Value(&m.teamName), + huh.NewInput(). + Title("管理员密码"). + Description("留空时自动生成随机密码。"). + Value(&m.teamPassword), + )) +} + +func validateAbsPath(value string) error { + if strings.TrimSpace(value) == "" { + return nil + } + if !filepath.IsAbs(strings.TrimSpace(value)) { + return fmt.Errorf("请输入绝对路径") + } + return nil +} + +func (m *centerModel) nextAfterForm() (tea.Model, tea.Cmd) { + switch m.step { + case centerStepConfirmDocker: + if !m.installDocker { + m.step = centerStepDone + m.err = fmt.Errorf("已取消安装 Docker/Compose") + return m, nil + } + m.step = centerStepForm + m.form = m.centerForm() + return m, m.form.Init() + case centerStepForm: + m.installDir = strings.TrimSpace(m.installDir) + if m.installDir == "" { + m.installDir = defaultCenterInstallDir + } + m.accessHost = strings.TrimSpace(m.accessHost) + m.nginxPort = strings.TrimSpace(m.nginxPort) + m.teamEmail = strings.TrimSpace(m.teamEmail) + m.teamName = strings.TrimSpace(m.teamName) + m.teamPassword = strings.TrimSpace(m.teamPassword) + m.step = centerStepRun + m.action = "安装中心端" + return m, m.installCenterFlow + } + return m, nil +} + +func (m *centerModel) installCenterFlow() tea.Msg { + pkgDir := packageDir() + if !m.status.Ready() { + if err := installer.InstallDockerFromLocalBundle(m.ctx, m.runner, centerDockerPlan(pkgDir)); err != nil { + return centerDoneMsg{err: err} + } + status := installer.CheckDockerStatus(m.ctx, m.runner) + if !status.Ready() { + return centerDoneMsg{err: fmt.Errorf("Docker 环境仍未就绪")} + } + } + env, err := installer.NewCenterEnv(m.centerEnvInput()) + if err != nil { + return centerDoneMsg{err: err} + } + if err := installer.PrepareCenterFiles(m.ctx, m.runner, installer.CenterFilesPlan{ + PackageDir: pkgDir, + WorkDir: m.installDir, + Env: env, + }); err != nil { + return centerDoneMsg{err: err} + } + plan := centerPlan(m.installDir, m.accessHost) + if err := installer.GenerateSelfSignedTLS(plan.TLS); err != nil { + return centerDoneMsg{err: err} + } + if err := installer.InstallCenter(m.ctx, m.runner, plan); err != nil { + return centerDoneMsg{err: err} + } + return centerDoneMsg{result: centerInstallResult{ + URL: centerAccessURL(env.AccessHost, env.NginxPort), + AdminEmail: env.TeamEmail, + AdminPassword: env.TeamPassword, + }} +} + +func centerDockerPlan(packageDir string) installer.DockerInstallPlan { + return installer.DockerInstallPlan{ + WorkDir: "/tmp/monkeycode-installer", + BundleFile: filepath.Join(packageDir, "docker.tgz"), + } +} + +func centerPlan(workDir, accessHost string) installer.CenterInstallPlan { + return installer.CenterInstallPlan{ + WorkDir: workDir, + ComposeFile: filepath.Join(workDir, "docker-compose.yml"), + EnvFile: filepath.Join(workDir, ".env"), + TLS: installer.TLSPlan{ + Host: accessHost, + CertFile: filepath.Join(workDir, "tls", "server.crt"), + KeyFile: filepath.Join(workDir, "tls", "server.key"), + }, + Images: scanImages(filepath.Join(workDir, "images")), + } +} + +func (m *centerModel) centerEnvInput() installer.CenterEnvInput { + return installer.CenterEnvInput{ + AccessHost: m.accessHost, + NginxPort: m.nginxPort, + TeamEmail: m.teamEmail, + TeamName: m.teamName, + TeamPassword: m.teamPassword, + } +} + +func centerAccessURL(host, port string) string { + if port == "" || port == "80" { + return "http://" + host + } + return "http://" + host + ":" + port +} + +func packageDir() string { + exe, err := os.Executable() + if err != nil { + return "." + } + return filepath.Dir(exe) +} diff --git a/backend/cmd/installer/center_test.go b/backend/cmd/installer/center_test.go new file mode 100644 index 00000000..f75dab40 --- /dev/null +++ b/backend/cmd/installer/center_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/chaitin/MonkeyCode/backend/pkg/installer" +) + +func TestCenterPlanUsesSelectedInstallDirAndAccessHost(t *testing.T) { + plan := centerPlan("/data/monkeycode-ai", "192.168.1.10") + + if plan.WorkDir != "/data/monkeycode-ai" { + t.Fatalf("WorkDir = %q", plan.WorkDir) + } + if plan.ComposeFile != "/data/monkeycode-ai/docker-compose.yml" { + t.Fatalf("ComposeFile = %q", plan.ComposeFile) + } + if plan.EnvFile != "/data/monkeycode-ai/.env" { + t.Fatalf("EnvFile = %q", plan.EnvFile) + } + if plan.TLS.Host != "192.168.1.10" { + t.Fatalf("TLS.Host = %q", plan.TLS.Host) + } + if plan.TLS.CertFile != "/data/monkeycode-ai/tls/server.crt" { + t.Fatalf("TLS.CertFile = %q", plan.TLS.CertFile) + } +} + +func TestCenterDockerPlanUsesLocalBundle(t *testing.T) { + plan := centerDockerPlan("/pkg") + + if plan.BundleFile != filepath.Join("/pkg", "docker.tgz") { + t.Fatalf("BundleFile = %q", plan.BundleFile) + } + if plan.WorkDir != "/tmp/monkeycode-installer" { + t.Fatalf("WorkDir = %q", plan.WorkDir) + } +} + +func TestCenterModelDefaults(t *testing.T) { + m := newCenterModel(t.Context()) + + if m.installDir != defaultCenterInstallDir { + t.Fatalf("installDir = %q", m.installDir) + } + if m.nginxPort != "80" { + t.Fatalf("nginxPort = %q", m.nginxPort) + } + if m.teamName != "MonkeyCode" { + t.Fatalf("teamName = %q", m.teamName) + } + if m.runner == nil { + t.Fatal("runner is nil") + } +} + +func TestCenterEnvInputUsesModelValues(t *testing.T) { + m := newCenterModel(t.Context()) + m.accessHost = "example.com" + m.nginxPort = "8080" + m.teamEmail = "admin@example.com" + m.teamName = "Example" + m.teamPassword = "secret" + + input := m.centerEnvInput() + + if input.AccessHost != "example.com" { + t.Fatalf("AccessHost = %q", input.AccessHost) + } + if input.NginxPort != "8080" { + t.Fatalf("NginxPort = %q", input.NginxPort) + } + if input.TeamEmail != "admin@example.com" { + t.Fatalf("TeamEmail = %q", input.TeamEmail) + } + if input.TeamName != "Example" { + t.Fatalf("TeamName = %q", input.TeamName) + } + if input.TeamPassword != "secret" { + t.Fatalf("TeamPassword = %q", input.TeamPassword) + } +} + +func TestCenterSuccessViewPrintsAccessAndAdmin(t *testing.T) { + m := newCenterModel(t.Context()) + m.step = centerStepDone + m.action = "安装中心端" + m.result = centerInstallResult{ + URL: "http://example.com:8080", + AdminEmail: "admin@example.com", + AdminPassword: "secret", + } + + view := m.View() + + for _, want := range []string{"http://example.com:8080", "admin@example.com", "secret"} { + if !strings.Contains(view, want) { + t.Fatalf("view missing %q: %q", want, view) + } + } +} + +var _ = installer.CenterInstallPlan{} diff --git a/backend/cmd/installer/main.go b/backend/cmd/installer/main.go new file mode 100644 index 00000000..81db9779 --- /dev/null +++ b/backend/cmd/installer/main.go @@ -0,0 +1,531 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/huh" + "github.com/charmbracelet/lipgloss" + + "github.com/chaitin/MonkeyCode/backend/pkg/installer" +) + +var ( + titleStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("212")) + okStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("42")) + warnStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("214")) + errStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("196")) +) + +type step int + +const ( + stepCheck step = iota + stepHome + stepInstallDir + stepConfirmDocker + stepUninstallDir + stepConfirmUninstall + stepRun + stepDone +) + +const defaultInstallDir = "/data/monkeycode_runner" + +type homeAction string + +const ( + homeActionInstall homeAction = "install" + homeActionUninstall homeAction = "uninstall" + homeActionExit homeAction = "exit" +) + +type installerMode string + +const ( + modeCenter installerMode = "center" + modeHost installerMode = "host" +) + +type model struct { + ctx context.Context + runner installer.Runner + spinner spinner.Model + prog progress.Model + config installConfig + step step + status installer.DockerStatus + form *huh.Form + home homeAction + installDir string + installDocker bool + uninstallHost bool + action string + detail string + err error + sender *programSender +} + +type statusMsg installer.DockerStatus +type actionDoneMsg struct{ err error } +type progressMsg struct { + label string + progress installer.DownloadProgress +} + +type programSender struct { + program *tea.Program +} + +func (s *programSender) Send(msg tea.Msg) { + if s.program != nil { + s.program.Send(msg) + } +} + +func newModel(ctx context.Context) *model { + sp := spinner.New() + sp.Spinner = spinner.Dot + return &model{ + ctx: ctx, + runner: installer.CommandRunner{}, + spinner: sp, + prog: progress.New(progress.WithWidth(46)), + config: loadConfig(), + step: stepCheck, + sender: &programSender{}, + } +} + +func parseMode(args []string) (installerMode, error) { + if len(args) <= 1 { + return modeHost, nil + } + switch installerMode(args[1]) { + case modeCenter: + return modeCenter, nil + case modeHost: + return modeHost, nil + default: + return "", fmt.Errorf("unknown installer mode %q, expected center or host", args[1]) + } +} + +func (m *model) setProgram(p *tea.Program) { + m.sender.program = p +} + +func (m *model) Init() tea.Cmd { + return tea.Batch(m.spinner.Tick, m.checkDocker) +} + +func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c": + return m, tea.Quit + } + case statusMsg: + m.status = installer.DockerStatus(msg) + m.step = stepHome + m.form = m.homeForm() + return m, m.form.Init() + case progressMsg: + m.detail = msg.label + if msg.progress.Total <= 0 { + return m, nil + } + return m, m.prog.SetPercent(msg.progress.Percent()) + case actionDoneMsg: + m.err = msg.err + m.step = stepDone + if msg.err == nil { + return m, tea.Quit + } + return m, nil + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case progress.FrameMsg: + pm, cmd := m.prog.Update(msg) + m.prog = pm.(progress.Model) + return m, cmd + } + if m.form != nil { + var cmd tea.Cmd + form, formCmd := m.form.Update(msg) + m.form = form.(*huh.Form) + cmd = formCmd + if m.form.State == huh.StateCompleted { + m.form = nil + return m.nextAfterForm() + } + if m.form.State == huh.StateAborted { + return m, tea.Quit + } + return m, cmd + } + return m, nil +} + +func (m *model) View() string { + var b strings.Builder + b.WriteString(titleStyle.Render("MonkeyCode 安装器")) + b.WriteString("\n\n") + + switch m.step { + case stepCheck: + b.WriteString(m.spinner.View()) + b.WriteString(" 正在检测 Docker 环境...\n") + case stepHome, stepInstallDir, stepConfirmDocker, stepUninstallDir, stepConfirmUninstall: + b.WriteString(renderStatus(m.status)) + b.WriteString("\n") + if m.form != nil { + b.WriteString(m.form.View()) + b.WriteString("\n") + } + case stepRun: + b.WriteString(m.spinner.View()) + b.WriteString(" " + m.action + "...\n") + if m.detail != "" { + b.WriteString(m.detail) + b.WriteString("\n") + } + b.WriteString(m.prog.View()) + b.WriteString("\n") + case stepDone: + if m.err != nil { + b.WriteString(errStyle.Render(m.action + "失败: " + m.err.Error())) + } else { + b.WriteString(okStyle.Render(m.action + "完成。")) + } + b.WriteString("\n") + } + return b.String() +} + +func (m *model) checkDocker() tea.Msg { + return statusMsg(installer.CheckDockerStatus(m.ctx, m.runner)) +} + +func (m *model) homeForm() *huh.Form { + return huh.NewForm( + huh.NewGroup( + huh.NewSelect[homeAction](). + Title("请选择操作"). + Options(homeOptions()...). + Value(&m.home), + ), + ) +} + +func homeOptions() []huh.Option[homeAction] { + return []huh.Option[homeAction]{ + huh.NewOption("安装", homeActionInstall), + huh.NewOption("卸载", homeActionUninstall), + huh.NewOption("退出", homeActionExit), + } +} + +func (m *model) installDirForm() *huh.Form { + if strings.TrimSpace(m.installDir) == "" { + m.installDir = defaultInstallDir + } + return huh.NewForm( + huh.NewGroup( + huh.NewInput(). + Title("安装目录"). + Description("宿主机服务文件、镜像包和 docker-compose.yml 将放在该目录。"). + Placeholder(defaultInstallDir). + Value(&m.installDir). + Validate(validateAbsPath), + ), + ) +} + +func (m *model) confirmDockerForm() *huh.Form { + return huh.NewForm( + huh.NewGroup( + huh.NewConfirm(). + Title("安装 Docker/Compose"). + Description("当前 Docker 环境不完整，将下载 Docker 安装包并安装静态二进制。"). + Affirmative("开始安装"). + Negative("取消"). + Value(&m.installDocker), + ), + ) +} + +func (m *model) confirmUninstallForm() *huh.Form { + return huh.NewForm( + huh.NewGroup( + huh.NewConfirm(). + Title("卸载宿主机"). + Description("将停止 MonkeyCode 宿主机服务并删除安装目录，不会卸载 Docker/Compose 或删除镜像。"). + Affirmative("确认卸载"). + Negative("取消"). + Value(&m.uninstallHost), + ), + ) +} + +func (m *model) nextAfterForm() (tea.Model, tea.Cmd) { + switch m.step { + case stepHome: + switch m.home { + case homeActionInstall: + m.action = "安装宿主机" + m.detail = "" + m.err = nil + m.installDir = defaultInstallDir + if !m.status.Ready() { + m.step = stepConfirmDocker + m.installDocker = true + m.form = m.confirmDockerForm() + return m, m.form.Init() + } + m.step = stepInstallDir + m.form = m.installDirForm() + return m, m.form.Init() + case homeActionUninstall: + m.action = "卸载宿主机" + m.detail = "" + m.err = nil + m.installDir = defaultInstallDir + m.step = stepUninstallDir + m.form = m.installDirForm() + return m, m.form.Init() + case homeActionExit: + return m, tea.Quit + } + case stepConfirmDocker: + if !m.installDocker { + m.step = stepDone + m.err = fmt.Errorf("已取消安装 Docker/Compose") + return m, nil + } + m.step = stepInstallDir + m.form = m.installDirForm() + return m, m.form.Init() + case stepInstallDir: + m.installDir = strings.TrimSpace(m.installDir) + if m.installDir == "" { + m.installDir = defaultInstallDir + } + m.step = stepRun + m.detail = "准备安装流程" + return m, m.installHostFlow + case stepUninstallDir: + m.installDir = strings.TrimSpace(m.installDir) + if m.installDir == "" { + m.installDir = defaultInstallDir + } + m.uninstallHost = false + m.step = stepConfirmUninstall + m.form = m.confirmUninstallForm() + return m, m.form.Init() + case stepConfirmUninstall: + if !m.uninstallHost { + m.step = stepDone + m.err = fmt.Errorf("已取消卸载宿主机") + return m, nil + } + m.step = stepRun + m.detail = "准备卸载流程" + return m, m.uninstallHostFlow + } + return m, nil +} + +func (m *model) installHostFlow() tea.Msg { + if !m.status.Ready() { + plan, err := m.config.dockerPlan() + if err != nil { + return actionDoneMsg{err: err} + } + if err := installer.InstallDockerWithProgress(m.ctx, m.runner, plan, m.sendProgress("下载 Docker 安装包")); err != nil { + return actionDoneMsg{err: err} + } + status := installer.CheckDockerStatus(m.ctx, m.runner) + if !status.Ready() { + return actionDoneMsg{err: fmt.Errorf("Docker 环境仍未就绪")} + } + } + + bundlePlan, err := m.config.hostBundlePlan(m.installDir) + if err != nil { + return actionDoneMsg{err: err} + } + if err := installer.PrepareHostBundleWithProgress(m.ctx, m.runner, bundlePlan, m.sendProgress("下载宿主机安装包")); err != nil { + return actionDoneMsg{err: err} + } + return actionDoneMsg{err: installer.InstallHost(m.ctx, m.runner, hostPlan(m.installDir))} +} + +func (m *model) uninstallHostFlow() tea.Msg { + return actionDoneMsg{err: installer.UninstallHost(m.ctx, m.runner, hostPlan(m.installDir))} +} + +func (m *model) sendProgress(label string) installer.ProgressFunc { + return func(p installer.DownloadProgress) { + m.sender.Send(progressMsg{label: label, progress: p}) + } +} + +func renderStatus(status installer.DockerStatus) string { + lines := []string{} + if status.DockerInstalled { + lines = append(lines, okStyle.Render("Docker: 已安装 "+status.DockerVersion)) + } else { + lines = append(lines, errStyle.Render("Docker: 未安装")) + } + if status.ComposeInstalled { + lines = append(lines, okStyle.Render("Docker Compose: 已安装 "+status.ComposeVersion)) + } else { + lines = append(lines, errStyle.Render("Docker Compose: 未安装")) + } + if status.DaemonRunning { + lines = append(lines, okStyle.Render("Docker Daemon: 运行中 "+status.DaemonVersion)) + } else { + lines = append(lines, warnStyle.Render("Docker Daemon: 未运行")) + } + return strings.Join(lines, "\n") +} + +func hostPlan(workDir string) installer.HostInstallPlan { + return installer.HostInstallPlan{ + WorkDir: workDir, + ComposeFile: filepath.Join(workDir, "docker-compose.yml"), + EnvFile: filepath.Join(workDir, ".env"), + Token: os.Getenv("MCAI_HOST_TOKEN"), + GrpcURL: os.Getenv("MCAI_TASKFLOW_GRPC_URL"), + Images: scanImages(filepath.Join(workDir, "images")), + } +} + +type installConfig struct { + BaseURL string + DockerBundlePath string + HostBundlePath string +} + +func loadConfig() installConfig { + arch := installerArch() + return installConfig{ + BaseURL: valueOrDefault(os.Getenv("MCAI_BASE_URL"), "http://localhost"), + DockerBundlePath: valueOrDefault(os.Getenv("MCAI_DOCKER_BUNDLE_PATH"), "/static/installer/"+arch+"/docker.tgz"), + HostBundlePath: valueOrDefault(os.Getenv("MCAI_HOST_BUNDLE_PATH"), "/static/installer/"+arch+"/host.tgz"), + } +} + +func installerArch() string { + switch runtime.GOARCH { + case "arm64": + return "aarch64" + default: + return "x86_64" + } +} + +func (c installConfig) dockerPlan() (installer.DockerInstallPlan, error) { + bundleURL, err := installer.BundleURL(c.BaseURL, c.DockerBundlePath) + if err != nil { + return installer.DockerInstallPlan{}, err + } + return installer.DockerInstallPlan{ + BundleURL: bundleURL, + WorkDir: "/tmp/monkeycode-installer", + BundleFile: "/tmp/monkeycode-installer/docker.tgz", + }, nil +} + +func (c installConfig) hostBundlePlan(workDir string) (installer.HostBundlePlan, error) { + bundleURL, err := installer.BundleURL(c.BaseURL, c.HostBundlePath) + if err != nil { + return installer.HostBundlePlan{}, err + } + return installer.HostBundlePlan{ + BundleURL: bundleURL, + BundleFile: "/tmp/monkeycode-host.tgz", + WorkDir: workDir, + }, nil +} + +func valueOrDefault(value, fallback string) string { + if value != "" { + return value + } + return fallback +} + +func scanImages(dir string) []installer.ImageArchive { + patterns := []struct { + glob string + compressed bool + }{ + {filepath.Join(dir, "*.tar"), false}, + {filepath.Join(dir, "*.tar.gz"), true}, + {filepath.Join(dir, "*.tgz"), true}, + } + images := []installer.ImageArchive{} + for _, pattern := range patterns { + matches, _ := filepath.Glob(pattern.glob) + for _, match := range matches { + if !isImageArchive(match) { + continue + } + images = append(images, installer.ImageArchive{Path: match, Compressed: pattern.compressed}) + } + } + sort.Slice(images, func(i, j int) bool { + return images[i].Path < images[j].Path + }) + return images +} + +func isImageArchive(file string) bool { + base := filepath.Base(file) + if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "._") || base == ".DS_Store" { + return false + } + info, err := os.Stat(file) + if err != nil { + return false + } + return !info.IsDir() +} + +func main() { + mode, err := parseMode(os.Args) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + var m tea.Model + switch mode { + case modeCenter: + m = newCenterModel(context.Background()) + case modeHost: + m = newModel(context.Background()) + } + + p := tea.NewProgram(m) + if senderAware, ok := m.(interface{ setProgram(*tea.Program) }); ok { + senderAware.setProgram(p) + } + if _, err := p.Run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/backend/cmd/installer/main_test.go b/backend/cmd/installer/main_test.go new file mode 100644 index 00000000..630775c5 --- /dev/null +++ b/backend/cmd/installer/main_test.go @@ -0,0 +1,249 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" + + "github.com/chaitin/MonkeyCode/backend/pkg/installer" +) + +func TestInstallerModeDefaultsToHost(t *testing.T) { + mode, err := parseMode([]string{"installer"}) + if err != nil { + t.Fatal(err) + } + if mode != modeHost { + t.Fatalf("mode = %q", mode) + } +} + +func TestInstallerModeAcceptsCenterAndHost(t *testing.T) { + for _, tc := range []struct { + args []string + want installerMode + }{ + {[]string{"installer", "center"}, modeCenter}, + {[]string{"installer", "host"}, modeHost}, + } { + got, err := parseMode(tc.args) + if err != nil { + t.Fatalf("parseMode(%v): %v", tc.args, err) + } + if got != tc.want { + t.Fatalf("parseMode(%v) = %q, want %q", tc.args, got, tc.want) + } + } +} + +func TestInstallerModeRejectsUnknownMode(t *testing.T) { + _, err := parseMode([]string{"installer", "agent"}) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "unknown installer mode") { + t.Fatalf("err = %v", err) + } +} + +func TestHomeOptionsOnlyInstallAndExit(t *testing.T) { + options := homeOptions() + + if len(options) != 3 { + t.Fatalf("len(options) = %d", len(options)) + } + if options[0].Key != "安装" || options[0].Value != homeActionInstall { + t.Fatalf("first option = %#v", options[0]) + } + if options[1].Key != "卸载" || options[1].Value != homeActionUninstall { + t.Fatalf("second option = %#v", options[1]) + } + if options[2].Key != "退出" || options[2].Value != homeActionExit { + t.Fatalf("third option = %#v", options[2]) + } +} + +func TestHomeSelectionExitsOnSingleSubmit(t *testing.T) { + m := newModel(t.Context()) + m.status = installer.DockerStatus{DockerInstalled: true, ComposeInstalled: true, DaemonRunning: true} + m.step = stepHome + m.form = m.homeForm() + _ = m.form.Init() + + updated, cmd := runUpdate(m, tea.KeyMsg{Type: tea.KeyDown}) + updated, cmd = runUpdate(updated, tea.KeyMsg{Type: tea.KeyDown}) + updated, cmd = runUpdate(updated, tea.KeyMsg{Type: tea.KeyEnter}) + + if cmd == nil { + t.Fatal("expected quit command") + } + if msg := cmd(); msg != tea.Quit() { + t.Fatalf("cmd() = %#v", msg) + } +} + +func TestHomeSelectionStartsUninstallOnSingleSubmit(t *testing.T) { + m := newModel(t.Context()) + m.status = installer.DockerStatus{DockerInstalled: true, ComposeInstalled: true, DaemonRunning: true} + m.step = stepHome + m.form = m.homeForm() + _ = m.form.Init() + + updated, _ := runUpdate(m, tea.KeyMsg{Type: tea.KeyDown}) + updated, _ = runUpdate(updated, tea.KeyMsg{Type: tea.KeyEnter}) + got := updated.(*model) + + if got.step != stepUninstallDir { + t.Fatalf("step = %v", got.step) + } + if got.form == nil { + t.Fatal("form is nil") + } +} + +func TestUninstallDirSelectionAsksForConfirmation(t *testing.T) { + m := newModel(t.Context()) + m.step = stepUninstallDir + m.installDir = "/data/monkeycode_runner" + m.form = m.installDirForm() + _ = m.form.Init() + + updated, _ := runUpdate(m, tea.KeyMsg{Type: tea.KeyEnter}) + got := updated.(*model) + + if got.step != stepConfirmUninstall { + t.Fatalf("step = %v", got.step) + } + if got.form == nil { + t.Fatal("form is nil") + } + if got.installDir != "/data/monkeycode_runner" { + t.Fatalf("installDir = %q", got.installDir) + } +} + +func TestHomeSelectionStartsInstallOnSingleSubmit(t *testing.T) { + m := newModel(t.Context()) + m.status = installer.DockerStatus{DockerInstalled: true, ComposeInstalled: true, DaemonRunning: true} + m.step = stepHome + m.form = m.homeForm() + _ = m.form.Init() + + updated, _ := runUpdate(m, tea.KeyMsg{Type: tea.KeyEnter}) + got := updated.(*model) + + if got.step != stepInstallDir { + t.Fatalf("step = %v", got.step) + } + if got.form == nil { + t.Fatal("form is nil") + } +} + +func TestInstallSuccessQuits(t *testing.T) { + m := newModel(t.Context()) + m.step = stepRun + m.action = "安装宿主机" + + _, cmd := m.Update(actionDoneMsg{}) + + if cmd == nil { + t.Fatal("expected quit command") + } + if msg := cmd(); msg != tea.Quit() { + t.Fatalf("cmd() = %#v", msg) + } +} + +func TestInstallFailureStaysOnDone(t *testing.T) { + m := newModel(t.Context()) + m.step = stepRun + m.action = "安装宿主机" + err := os.ErrNotExist + + updated, cmd := m.Update(actionDoneMsg{err: err}) + got := updated.(*model) + + if cmd != nil { + t.Fatal("expected no quit command") + } + if got.step != stepDone { + t.Fatalf("step = %v", got.step) + } + if got.err != err { + t.Fatalf("err = %v", got.err) + } +} + +func runUpdate(m tea.Model, msg tea.Msg) (tea.Model, tea.Cmd) { + updated, cmd := m.Update(msg) + for cmd != nil { + next := cmd() + if next == tea.Quit() { + return updated, cmd + } + updated, cmd = updated.Update(next) + } + return updated, nil +} + +func TestHostPlanUsesSelectedInstallDir(t *testing.T) { + plan := hostPlan("/data/monkeycode_runner") + + if plan.WorkDir != "/data/monkeycode_runner" { + t.Fatalf("WorkDir = %q", plan.WorkDir) + } + if plan.ComposeFile != "/data/monkeycode_runner/docker-compose.yml" { + t.Fatalf("ComposeFile = %q", plan.ComposeFile) + } + if plan.EnvFile != "/data/monkeycode_runner/.env" { + t.Fatalf("EnvFile = %q", plan.EnvFile) + } +} + +func TestHostBundlePlanUsesSelectedInstallDir(t *testing.T) { + cfg := installConfig{ + BaseURL: "http://server", + HostBundlePath: "/static/installer/x86_64/host.tgz", + } + + plan, err := cfg.hostBundlePlan("/data/monkeycode_runner") + if err != nil { + t.Fatal(err) + } + + if plan.WorkDir != "/data/monkeycode_runner" { + t.Fatalf("WorkDir = %q", plan.WorkDir) + } + if plan.BundleFile != filepath.Join("/tmp", "monkeycode-host.tgz") { + t.Fatalf("BundleFile = %q", plan.BundleFile) + } +} + +func TestScanImagesSkipsMacOSMetadataFiles(t *testing.T) { + dir := t.TempDir() + imageDir := filepath.Join(dir, "images") + if err := os.Mkdir(imageDir, 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"orchestrator.tgz", "._orchestrator.tgz", ".DS_Store", ".hidden.tgz"} { + if err := os.WriteFile(filepath.Join(imageDir, name), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + images := scanImages(imageDir) + + if len(images) != 1 { + t.Fatalf("len(images) = %d, images=%#v", len(images), images) + } + if images[0].Path != filepath.Join(imageDir, "orchestrator.tgz") { + t.Fatalf("image path = %q", images[0].Path) + } + if !images[0].Compressed { + t.Fatal("image should be compressed") + } +} diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index a8131621..e06b0c80 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -47,10 +47,10 @@ func main() { } // 注册业务模块 - if err := biz.RegisterAll(injector); err != nil { - l.Error("failed to register biz", "error", err) - os.Exit(1) - } + biz.RegisterAll(injector) + biz.RegisterOpenSource(injector) + biz.InvokeAll(injector) + biz.InvokeOpenSource(injector) // 获取 web 实例并启动服务 w := do.MustInvoke[*web.Web](injector) diff --git a/backend/config/config.go b/backend/config/config.go index 88c1ec1e..61309a4f 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -43,17 +43,20 @@ type Config struct { AdminToken string `mapstructure:"admin_token"` Proxies []string `mapstructure:"proxies"` - TaskFlow TaskFlow `mapstructure:"taskflow"` - MCPHub MCPHub `mapstructure:"mcp_hub"` - PublicHost PublicHost `mapstructure:"public_host"` - Task Task `mapstructure:"task"` - TaskSummary TaskSummary `mapstructure:"task_summary"` - Loki Loki `mapstructure:"loki"` - ClickHouse ClickHouse `mapstructure:"clickhouse"` - LLM LLM `mapstructure:"llm"` - Notify Notify `mapstructure:"notify"` - VMIdle VMIdle `mapstructure:"vm_idle"` - Attachment Attachment `mapstructure:"attachment"` + TaskFlow TaskFlow `mapstructure:"taskflow"` + MCPHub MCPHub `mapstructure:"mcp_hub"` + PublicHost PublicHost `mapstructure:"public_host"` + Task Task `mapstructure:"task"` + TaskSummary TaskSummary `mapstructure:"task_summary"` + Loki Loki `mapstructure:"loki"` + ClickHouse ClickHouse `mapstructure:"clickhouse"` + LLM LLM `mapstructure:"llm"` + Notify Notify `mapstructure:"notify"` + VMIdle VMIdle `mapstructure:"vm_idle"` + Attachment Attachment `mapstructure:"attachment"` + ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"` + StaticFiles StaticFilesConfig `mapstructure:"static_files"` + HostInstaller HostInstaller `mapstructure:"host_installer"` // Context7 API 配置 Context7ApiKey string `mapstructure:"context7_api_key"` @@ -105,6 +108,36 @@ type Attachment struct { AllowedURLPrefixes []string `mapstructure:"allowed_url_prefixes"` } +type ObjectStorageConfig struct { + Enabled bool `mapstructure:"enabled"` + Provider string `mapstructure:"provider"` + ForcePathStyle bool `mapstructure:"force_path_style"` + InitBucket bool `mapstructure:"init_bucket"` + PresignExpires string `mapstructure:"presign_expires"` + Endpoint string `mapstructure:"endpoint"` + AccessEndpoint string `mapstructure:"access_endpoint"` + AccessKey string `mapstructure:"access_key"` + AccessKeySecret string `mapstructure:"access_key_secret"` + Bucket string `mapstructure:"bucket"` + Region string `mapstructure:"region"` + MaxSize int64 `mapstructure:"max_size"` + AvatarPrefix string `mapstructure:"avatar_prefix"` + SpecPrefix string `mapstructure:"spec_prefix"` + RepoPrefix string `mapstructure:"repo_prefix"` + TempPrefix string `mapstructure:"temp_prefix"` +} + +type StaticFilesConfig struct { + Enabled bool `mapstructure:"enabled"` + Dir string `mapstructure:"dir"` + RoutePrefix string `mapstructure:"route_prefix"` +} + +type HostInstaller struct { + Mode string `mapstructure:"mode"` + BundlePath string `mapstructure:"bundle_path"` +} + // Task 任务相关配置 type Task struct { LogLimit int `mapstructure:"log_limit"` // Loki tail 日志 limit @@ -193,7 +226,7 @@ func Init(dir string) (*Config, error) { v.SetDefault("debug", false) v.SetDefault("server.addr", ":8888") - v.SetDefault("server.base_url", "http://localhost:8888") + v.SetDefault("server.base_url", "") v.SetDefault("loki.addr", "http://monkeycode-ai-loki:3100") v.SetDefault("clickhouse.addr", "") v.SetDefault("clickhouse.database", "") @@ -236,6 +269,28 @@ func Init(dir string) (*Config, error) { v.SetDefault("mcp_hub.url", "") v.SetDefault("mcp_hub.token", "") v.SetDefault("attachment.allowed_url_prefixes", []string{}) + v.SetDefault("object_storage.enabled", false) + v.SetDefault("object_storage.provider", "s3") + v.SetDefault("object_storage.force_path_style", true) + v.SetDefault("object_storage.init_bucket", false) + v.SetDefault("object_storage.presign_expires", "168h") + v.SetDefault("object_storage.endpoint", "http://monkeycode-ai-rustfs:9000") + v.SetDefault("object_storage.access_endpoint", "") + v.SetDefault("object_storage.access_key", "") + v.SetDefault("object_storage.access_key_secret", "") + v.SetDefault("object_storage.bucket", "monkeycode-ai") + v.SetDefault("object_storage.region", "us-east-1") + v.SetDefault("object_storage.max_size", 50<<20) + v.SetDefault("object_storage.avatar_prefix", "avatar") + v.SetDefault("object_storage.spec_prefix", "spec") + v.SetDefault("object_storage.repo_prefix", "repo") + v.SetDefault("object_storage.temp_prefix", "temp") + v.SetDefault("static_files.enabled", true) + v.SetDefault("static_files.dir", "/app/static") + v.SetDefault("static_files.route_prefix", "/static") + v.SetDefault("host_installer.mode", "online") + v.SetDefault("host_installer.bundle_path", "installer/{{.arch}}/host.tgz") + v.SetDefault("llm_proxy.base_url", "") v.SetConfigType("yaml") v.AddConfigPath(dir) diff --git a/backend/config/oss_config_test.go b/backend/config/oss_config_test.go new file mode 100644 index 00000000..65df99f3 --- /dev/null +++ b/backend/config/oss_config_test.go @@ -0,0 +1,60 @@ +package config + +import "testing" + +func TestObjectStorageDefaults(t *testing.T) { + t.Setenv("MCAI_OBJECT_STORAGE_ENABLED", "") + t.Setenv("MCAI_OBJECT_STORAGE_PROVIDER", "") + t.Setenv("MCAI_OBJECT_STORAGE_FORCE_PATH_STYLE", "") + t.Setenv("MCAI_OBJECT_STORAGE_PRESIGN_EXPIRES", "") + t.Setenv("MCAI_OBJECT_STORAGE_MAX_SIZE", "") + t.Setenv("MCAI_OBJECT_STORAGE_TEMP_PREFIX", "") + t.Setenv("MCAI_TASKFLOW_GRPC_URL", "") + + cfg, err := Init(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if cfg.ObjectStorage.Enabled { + t.Fatal("object_storage.enabled default = true, want false") + } + if cfg.Server.BaseURL != "" { + t.Fatalf("server.base_url = %q, want empty", cfg.Server.BaseURL) + } + if cfg.ObjectStorage.Provider != "s3" { + t.Fatalf("provider = %q, want s3", cfg.ObjectStorage.Provider) + } + if !cfg.ObjectStorage.ForcePathStyle { + t.Fatal("force_path_style default = false, want true") + } + if cfg.ObjectStorage.PresignExpires != "168h" { + t.Fatalf("presign_expires = %q, want 168h", cfg.ObjectStorage.PresignExpires) + } + if cfg.ObjectStorage.AccessEndpoint != "" { + t.Fatalf("access_endpoint = %q, want empty", cfg.ObjectStorage.AccessEndpoint) + } + if cfg.ObjectStorage.MaxSize != 50<<20 { + t.Fatalf("max_size = %d, want %d", cfg.ObjectStorage.MaxSize, 50<<20) + } + if cfg.ObjectStorage.TempPrefix != "temp" { + t.Fatalf("temp_prefix = %q", cfg.ObjectStorage.TempPrefix) + } + if cfg.TaskFlow.GrpcURL != "" { + t.Fatalf("taskflow.grpc_url = %q, want empty", cfg.TaskFlow.GrpcURL) + } + if !cfg.StaticFiles.Enabled { + t.Fatal("static_files.enabled default = false, want true") + } + if cfg.StaticFiles.Dir != "/app/static" { + t.Fatalf("static_files.dir = %q", cfg.StaticFiles.Dir) + } + if cfg.StaticFiles.RoutePrefix != "/static" { + t.Fatalf("static_files.route_prefix = %q", cfg.StaticFiles.RoutePrefix) + } + if cfg.HostInstaller.Mode != "online" { + t.Fatalf("host_installer.mode = %q", cfg.HostInstaller.Mode) + } + if cfg.HostInstaller.BundlePath != "installer/{{.arch}}/host.tgz" { + t.Fatalf("host_installer.bundle_path = %q", cfg.HostInstaller.BundlePath) + } +} diff --git a/backend/config/server/config.yaml.example b/backend/config/server/config.yaml.example index b92d3a39..7ac9cf20 100644 --- a/backend/config/server/config.yaml.example +++ b/backend/config/server/config.yaml.example @@ -1,6 +1,7 @@ debug: true server: addr: ":8888" + # 必须手动配置为外部可访问地址，用于安装脚本、回调、对象存储公开 URL 等场景。 base_url: "http://localhost:8888" database: @@ -55,3 +56,32 @@ clickhouse: max_open_conns: 64 max_idle_conns: 32 conn_max_lifetime: 3600 + +object_storage: + enabled: false + provider: "s3" + force_path_style: true + init_bucket: false + presign_expires: "168h" + + endpoint: "" + # 外部访问入口，例如 http://example.com/oss。留空时回退到 endpoint。 + access_endpoint: "" + access_key: "" + access_key_secret: "" + bucket: "" + region: "us-east-1" + max_size: 52428800 + avatar_prefix: "avatar" + spec_prefix: "spec" + repo_prefix: "repo" + temp_prefix: "temp" + +static_files: + enabled: true + dir: "/app/static" + route_prefix: "/static" + +host_installer: + mode: "online" + bundle_path: "installer/{{.arch}}/host.tgz" diff --git a/backend/consts/uploader.go b/backend/consts/uploader.go new file mode 100644 index 00000000..106aac4b --- /dev/null +++ b/backend/consts/uploader.go @@ -0,0 +1,9 @@ +package consts + +type UploadUsage string + +const ( + UploadUsageAvatar UploadUsage = "avatar" + UploadUsageSpec UploadUsage = "spec" + UploadUsageRepo UploadUsage = "repo" +) diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index e8b4db2c..f6223a66 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -1,21 +1,178 @@ -version: '3.8' - +name: monkeycode-ai services: - postgres: - image: postgres:16-alpine + db: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/postgres:17.4-alpine3.21 + container_name: monkeycode-ai-db + restart: always + init: true environment: - POSTGRES_USER: monkeycode - POSTGRES_PASSWORD: monkeycode - POSTGRES_DB: monkeycode - ports: - - "5432:5432" + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} volumes: - - pgdata:/var/lib/postgresql/data + - pg_data:/var/lib/postgresql/data + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.11" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"] + interval: 5s + timeout: 5s + retries: 5 redis: - image: redis:7-alpine + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/redis:8.0-alpine3.21 + container_name: monkeycode-ai-redis + restart: always + command: ["redis-server", "--requirepass", "${REDIS_PASSWORD}", "--appendonly", "yes", "--appendfilename", "appendonly.aof", "--save", "900 1", "--save", "300 10", "--save", "60 10000"] + volumes: + - redis_data:/data + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.12" + + clickhouse: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/clickhouse-server:26.3.9 + container_name: monkeycode-ai-clickhouse + restart: always + environment: + CLICKHOUSE_DB: ${CLICKHOUSE_DB} + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: '1' + volumes: + - ch_data:/var/lib/clickhouse + - ./logs/clickhouse:/var/log/clickhouse-server + ulimits: + nofile: + soft: 262144 + hard: 262144 + cap_add: + - SYS_NICE + - IPC_LOCK + healthcheck: + test: ["CMD-SHELL", "clickhouse-client --query 'SELECT 1' >/dev/null 2>&1 || exit 1"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + stop_grace_period: 2m + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.13" + + rustfs: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/rustfs:1.0.0-beta.2 + container_name: monkeycode-ai-rustfs + security_opt: + - "no-new-privileges:true" + environment: + # API 和控制台监听地址 + - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 + - RUSTFS_CONSOLE_ENABLE=true + # CORS 设置，控制台与 S3 API 都放开来源 + - RUSTFS_CORS_ALLOWED_ORIGINS=* + - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=* + - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY} + - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY} + # 日志级别 + - RUSTFS_OBS_LOGGER_LEVEL=info + volumes: + - rustfs_data:/data + - ./logs/rustfs:/app/logs + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.14" + restart: always + healthcheck: + test: ["CMD", "sh", "-c", "curl -f http://localhost:9000/health && curl -f http://localhost:9001/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + ingress: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/ingress:7f7f8b2 + container_name: monkeycode-ai-ingress + restart: always ports: - - "6379:6379" + - ${NGINX_PORT:-80}:80 + - 50443:50443 + volumes: + - ./tls:/etc/tls + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.15" + + taskflow: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/taskflow:fc0daba + container_name: monkeycode-ai-taskflow + restart: always + environment: + MCAI_DATABASE_MASTER: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@monkeycode-ai-db:5432/${POSTGRES_DB}?sslmode=disable&timezone=Asia/Shanghai + MCAI_REDIS_HOST: monkeycode-ai-redis + MCAI_REDIS_PASS: ${REDIS_PASSWORD} + MCAI_CLICKHOUSE_ADDR: monkeycode-ai-clickhouse:9000 + MCAI_CLICKHOUSE_DATABASE: ${CLICKHOUSE_DB} + MCAI_CLICKHOUSE_USERNAME: ${CLICKHOUSE_USER} + MCAI_CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + MCAI_TEMPLATE_PROJECT_ZIP_URL: http://${REMOTE_IP}:${NGINX_PORT:-80}/static/project-tpl.zip + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.16" + + frontend: + image: ghcr.1ms.run/chaitin/monkeycode-frontend:6048ca01cfd9663bd45121f06a3979d59eb6a6e2 + container_name: monkeycode-ai-frontend + restart: always + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.17" + + backend: + image: chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/backend:7f7f8b2 + container_name: monkeycode-ai-backend + restart: always + environment: + MCAI_LOGGER_LEVEL: debug + TASKFLOW_SERVER: http://monkeycode-ai-taskflow:8888 + MCAI_SERVER_BASE_URL: http://${REMOTE_IP}:${NGINX_PORT:-80} + MCAI_LLM_PROXY_BASE_URL: http://${REMOTE_IP}:${NGINX_PORT:-80} + MCAI_HOST_INSTALLER_MODE: offline + MCAI_DATABASE_MASTER: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@monkeycode-ai-db:5432/${POSTGRES_DB}?sslmode=disable&timezone=Asia/Shanghai + MCAI_REDIS_HOST: monkeycode-ai-redis + MCAI_REDIS_PASS: ${REDIS_PASSWORD} + MCAI_CLICKHOUSE_INIT_ENABLED: true + MCAI_CLICKHOUSE_ADDR: monkeycode-ai-clickhouse:9000 + MCAI_CLICKHOUSE_DATABASE: ${CLICKHOUSE_DB} + MCAI_CLICKHOUSE_USERNAME: ${CLICKHOUSE_USER} + MCAI_CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + MCAI_INIT_TEAM_EMAIL: ${TEAM_EMAIL} + MCAI_INIT_TEAM_NAME: ${TEAM_NAME} + MCAI_INIT_TEAM_PASSWORD: ${TEAM_PASSWORD} + MCAI_OBJECT_STORAGE_ENABLED: true + MCAI_OBJECT_STORAGE_INIT_BUCKET: true + MCAI_OBJECT_STORAGE_ACCESS_KEY: ${RUSTFS_ACCESS_KEY} + MCAI_OBJECT_STORAGE_ACCESS_KEY_SECRET: ${RUSTFS_SECRET_KEY} + MCAI_TASK_LLM_PROXY_ENABLED: false + MCAI_OBJECT_STORAGE_ACCESS_ENDPOINT: http://${REMOTE_IP}:${NGINX_PORT:-80}/oss + MCAI_TASKFLOW_GRPC_URL: ${REMOTE_IP}:50443 + volumes: + - ./static:/app/static + networks: + monkeycode: + ipv4_address: "${SUBNET_PREFIX:-10.100.50}.18" + +networks: + monkeycode: + ipam: + driver: default + config: + - subnet: "${SUBNET_PREFIX:-10.100.50}.0/24" volumes: - pgdata: + rustfs_data: + pg_data: + ch_data: + redis_data: \ No newline at end of file diff --git a/backend/domain/uploader.go b/backend/domain/uploader.go new file mode 100644 index 00000000..641e4f62 --- /dev/null +++ b/backend/domain/uploader.go @@ -0,0 +1,21 @@ +package domain + +import ( + "mime/multipart" + + "github.com/chaitin/MonkeyCode/backend/consts" +) + +type UploadReq struct { + Usage consts.UploadUsage `json:"usage" form:"usage" validate:"required,oneof=avatar spec repo"` + File *multipart.FileHeader `json:"file" form:"file"` +} + +type PresignReq struct { + Filename string `json:"filename" form:"filename" validate:"required"` +} + +type PresignResp struct { + UploadURL string `json:"upload_url"` + AccessURL string `json:"access_url"` +} diff --git a/backend/errcode/errcode.go b/backend/errcode/errcode.go index a6e63c57..ea4a2a25 100644 --- a/backend/errcode/errcode.go +++ b/backend/errcode/errcode.go @@ -43,7 +43,7 @@ var ( ErrVMIDRequired = web.NewErr(http.StatusOK, 10200, "err-vm-id-required") ErrVMNotBelongToUser = web.NewErr(http.StatusOK, 10201, "err-vm-not-belong-to-user") ErrPermisionDenied = web.NewErr(http.StatusOK, 10202, "err-permision-denied") - ErrInvalidInstallToken = web.NewErr(http.StatusOK, 10203, "err-host-id-required") + ErrInvalidInstallToken = web.NewErr(http.StatusOK, 10203, "err-invalid-token") ErrPublicHostNotFound = web.NewErr(http.StatusOK, 10204, "err-public-host-not-found") ErrHostOffline = web.NewErr(http.StatusOK, 10205, "err-host-offline") ErrVMExpired = web.NewErr(http.StatusOK, 10206, "err-vm-expired") diff --git a/backend/go.mod b/backend/go.mod index 843b6de9..6f019548 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -11,6 +11,14 @@ require ( github.com/alicebob/miniredis/v2 v2.35.0 github.com/aliyun/alibabacloud-nls-go-sdk v1.1.1 github.com/anthropics/anthropic-sdk-go v1.40.0 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/charmbracelet/bubbles v1.0.0 + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/huh v1.0.0 + github.com/charmbracelet/lipgloss v1.1.0 github.com/coder/websocket v1.8.14 github.com/gogo/protobuf v1.3.2 github.com/golang-migrate/migrate/v4 v4.19.0 @@ -40,12 +48,40 @@ require ( github.com/aliyun/alibaba-cloud-sdk-go v1.61.1376 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/aws/smithy-go v1.25.1 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect + github.com/catppuccin/go v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.4.1 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect + github.com/charmbracelet/x/ansi v0.11.6 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.9.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.5.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-faster/city v1.0.1 // indirect @@ -75,18 +111,26 @@ require ( github.com/klauspost/compress v1.18.3 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect github.com/nicksnyder/go-i18n/v2 v2.6.1 // indirect github.com/paulmach/orb v0.12.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/zerolog v1.34.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/satori/go.uuid v1.2.0 // indirect @@ -106,6 +150,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/gopher-lua v1.1.1 // indirect github.com/zclconf/go-cty v1.14.4 // indirect github.com/zclconf/go-cty-yaml v1.1.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index b02956bd..a0de5709 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -14,6 +14,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/GoYoko/web v1.6.0 h1:gwnErVfMSDKc8XwJIW9iiMBNuzwx1E3QwqPiGwEW76U= github.com/GoYoko/web v1.6.0/go.mod h1:MNOw+4KjmtRzUabIMqWK3t59yibnO1sDCp3EcLCmJVc= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ackcoder/go-cap v1.1.3 h1:rHIZEmyOM/KlXJQxGoy3UHpzpeUhw+V8qa/OoEaJR7A= @@ -32,6 +34,48 @@ github.com/anthropics/anthropic-sdk-go v1.40.0 h1:+lhHU2LdeRlVsazVXHswFMpWr2Q11S github.com/anthropics/anthropic-sdk-go v1.40.0/go.mod h1:d288C1L+m74OYuYBvc4UFtR1Q8J0gC55oYDh2t+XxdI= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= +github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= @@ -44,8 +88,46 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= +github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= +github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= +github.com/charmbracelet/huh v1.0.0 h1:wOnedH8G4qzJbmhftTqrpppyqHakl/zbbNdXIWJyIxw= +github.com/charmbracelet/huh v1.0.0/go.mod h1:5YVc+SlZ1IhQALxRPpkGwwEKftN/+OlJlnJYlDRFqN4= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8= +github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= +github.com/charmbracelet/x/conpty v0.1.0/go.mod h1:rMFsDJoDwVmiYM10aD4bH2XiRgwI7NYJtQgl5yskjEQ= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4= +github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/xpty v0.1.2 h1:Pqmu4TEJ8KeA9uSkISKMU3f+C1F6OGBn8ABuGlqCbtI= +github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJnzHI0Lq13Xzq4= +github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= +github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= +github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -53,6 +135,8 @@ github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -70,6 +154,10 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -187,6 +275,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -196,10 +286,16 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -213,6 +309,12 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ= github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -238,6 +340,8 @@ github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8A github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= @@ -301,6 +405,8 @@ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+x github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= @@ -347,6 +453,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= +golang.org/x/exp v0.0.0-20250813145105-42675adae3e6/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= @@ -373,6 +481,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/backend/pkg/installer/installer.go b/backend/pkg/installer/installer.go new file mode 100644 index 00000000..5d1df957 --- /dev/null +++ b/backend/pkg/installer/installer.go @@ -0,0 +1,559 @@ +package installer + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" +) + +var errNotFound = errors.New("command not found") + +type Runner interface { + Run(ctx context.Context, name string, args ...string) RunResult + RunShell(ctx context.Context, script string) RunResult +} + +type RunResult struct { + Stdout string + Stderr string + Err error +} + +type CommandRunner struct{} + +func (CommandRunner) Run(ctx context.Context, name string, args ...string) RunResult { + cmd := exec.CommandContext(ctx, name, args...) + out, err := cmd.Output() + if err != nil { + if _, ok := err.(*exec.Error); ok { + return RunResult{Err: errNotFound} + } + if exitErr, ok := err.(*exec.ExitError); ok { + return RunResult{Stderr: string(exitErr.Stderr), Err: err} + } + return RunResult{Err: err} + } + return RunResult{Stdout: string(out)} +} + +func (CommandRunner) RunShell(ctx context.Context, script string) RunResult { + return CommandRunner{}.Run(ctx, "sh", "-c", script) +} + +type DockerStatus struct { + DockerInstalled bool + DockerVersion string + ComposeInstalled bool + ComposeVersion string + DaemonRunning bool + DaemonVersion string +} + +func (s DockerStatus) Ready() bool { + return s.DockerInstalled && s.ComposeInstalled && s.DaemonRunning +} + +func CheckDockerStatus(ctx context.Context, r Runner) DockerStatus { + status := DockerStatus{} + if res := r.Run(ctx, "docker", "--version"); res.Err == nil { + status.DockerInstalled = true + status.DockerVersion = parseDockerVersion(res.Stdout) + } + if res := r.Run(ctx, "docker", "compose", "version"); res.Err == nil { + status.ComposeInstalled = true + status.ComposeVersion = parseComposeVersion(res.Stdout) + } + if res := r.Run(ctx, "docker", "info", "--format", "{{.ServerVersion}}"); res.Err == nil { + status.DaemonRunning = true + status.DaemonVersion = strings.TrimSpace(res.Stdout) + } + return status +} + +type HostInstallPlan struct { + WorkDir string + ComposeFile string + EnvFile string + Token string + GrpcURL string + Images []ImageArchive +} + +type CenterInstallPlan struct { + WorkDir string + ComposeFile string + EnvFile string + TLS TLSPlan + Images []ImageArchive +} + +type CenterFilesPlan struct { + PackageDir string + WorkDir string + Env CenterEnv +} + +type CenterEnvInput struct { + AccessHost string + NginxPort string + TeamEmail string + TeamName string + TeamPassword string +} + +type CenterEnv struct { + AccessHost string + NginxPort string + PostgresDB string + PostgresUser string + PostgresPassword string + RedisPassword string + ClickHouseDB string + ClickHouseUser string + ClickHousePassword string + RustFSAccessKey string + RustFSSecretKey string + TeamEmail string + TeamName string + TeamPassword string + SubnetPrefix string +} + +type DockerInstallPlan struct { + BundleURL string + WorkDir string + BundleFile string +} + +type HostBundlePlan struct { + BundleURL string + BundleFile string + WorkDir string +} + +type ImageArchive struct { + Path string + Compressed bool +} + +type DownloadProgress struct { + Downloaded int64 + Total int64 +} + +func (p DownloadProgress) Percent() float64 { + if p.Total <= 0 { + return 0 + } + percent := float64(p.Downloaded) / float64(p.Total) + if percent > 1 { + return 1 + } + return percent +} + +type ProgressFunc func(DownloadProgress) + +func InstallDocker(ctx context.Context, r Runner, plan DockerInstallPlan) error { + return InstallDockerWithProgress(ctx, r, plan, nil) +} + +func InstallDockerWithProgress(ctx context.Context, r Runner, plan DockerInstallPlan, progress ProgressFunc) error { + if res := r.Run(ctx, "mkdir", "-p", plan.WorkDir); res.Err != nil { + return fmt.Errorf("create docker install dir: %w", res.Err) + } + if err := DownloadFile(ctx, plan.BundleURL, plan.BundleFile, progress); err != nil { + return fmt.Errorf("download docker bundle: %w", err) + } + return installDockerBundle(ctx, r, plan) +} + +func InstallDockerFromLocalBundle(ctx context.Context, r Runner, plan DockerInstallPlan) error { + if res := r.Run(ctx, "mkdir", "-p", plan.WorkDir); res.Err != nil { + return fmt.Errorf("create docker install dir: %w", res.Err) + } + return installDockerBundle(ctx, r, plan) +} + +func installDockerBundle(ctx context.Context, r Runner, plan DockerInstallPlan) error { + if res := r.Run(ctx, "tar", "-zxf", plan.BundleFile, "-C", plan.WorkDir); res.Err != nil { + return fmt.Errorf("extract docker bundle: %w", res.Err) + } + if res := r.RunShell(ctx, fmt.Sprintf("install -m 0755 '%s'/docker/* /usr/local/bin/", shellQuote(plan.WorkDir))); res.Err != nil { + return fmt.Errorf("install docker binaries: %w", res.Err) + } + if res := r.Run(ctx, "install", "-m", "0755", plan.WorkDir+"/docker-compose", "/usr/local/lib/docker/cli-plugins/docker-compose"); res.Err != nil { + return fmt.Errorf("install docker compose: %w", res.Err) + } + if res := r.Run(ctx, "ln", "-sf", "/usr/local/lib/docker/cli-plugins/docker-compose", "/usr/local/bin/docker-compose"); res.Err != nil { + return fmt.Errorf("link docker compose: %w", res.Err) + } + if res := r.Run(ctx, "systemctl", "daemon-reload"); res.Err != nil { + return fmt.Errorf("reload systemd: %w", res.Err) + } + if res := r.Run(ctx, "systemctl", "enable", "--now", "docker"); res.Err != nil { + return fmt.Errorf("start docker: %w", res.Err) + } + return nil +} + +func PrepareHostBundle(ctx context.Context, r Runner, plan HostBundlePlan) error { + return PrepareHostBundleWithProgress(ctx, r, plan, nil) +} + +func PrepareHostBundleWithProgress(ctx context.Context, r Runner, plan HostBundlePlan, progress ProgressFunc) error { + if res := r.Run(ctx, "mkdir", "-p", plan.WorkDir); res.Err != nil { + return fmt.Errorf("create host install dir: %w", res.Err) + } + if err := DownloadFile(ctx, plan.BundleURL, plan.BundleFile, progress); err != nil { + return fmt.Errorf("download host bundle: %w", err) + } + if res := r.Run(ctx, "tar", "-zxf", plan.BundleFile, "-C", plan.WorkDir); res.Err != nil { + return fmt.Errorf("extract host bundle: %w", res.Err) + } + return nil +} + +func DownloadFile(ctx context.Context, sourceURL, dest string, progress ProgressFunc) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("unexpected status: %s", resp.Status) + } + + out, err := os.Create(dest) + if err != nil { + return err + } + defer out.Close() + + reader := &progressReader{ + reader: resp.Body, + total: resp.ContentLength, + progress: progress, + } + if _, err := io.Copy(out, reader); err != nil { + return err + } + if progress != nil { + progress(DownloadProgress{Downloaded: reader.downloaded, Total: resp.ContentLength}) + } + return nil +} + +type progressReader struct { + reader io.Reader + downloaded int64 + total int64 + progress ProgressFunc +} + +func (r *progressReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if n > 0 { + r.downloaded += int64(n) + if r.progress != nil { + r.progress(DownloadProgress{Downloaded: r.downloaded, Total: r.total}) + } + } + return n, err +} + +func BundleURL(baseURL, bundlePath string) (string, error) { + u, err := url.Parse(strings.TrimRight(baseURL, "/")) + if err != nil { + return "", err + } + u.Path = path.Join(u.Path, bundlePath) + return u.String(), nil +} + +func InstallHost(ctx context.Context, r Runner, plan HostInstallPlan) error { + if err := loadImages(ctx, r, plan.Images); err != nil { + return err + } + + if plan.EnvFile != "" && (plan.Token != "" || plan.GrpcURL != "") { + lines := []string{} + if plan.Token != "" { + lines = append(lines, "TOKEN="+plan.Token) + } + if plan.GrpcURL != "" { + lines = append(lines, "GRPC_URL="+plan.GrpcURL) + } + args := []string{"-c", fmt.Sprintf("printf '%%s\\n' %s >> '%s'", shellJoin(lines), shellQuote(plan.EnvFile))} + if res := r.Run(ctx, "sh", args...); res.Err != nil { + return fmt.Errorf("write host env: %w", res.Err) + } + } + + args := []string{"compose", "-f", plan.ComposeFile} + if plan.EnvFile != "" { + args = append(args, "--env-file", plan.EnvFile) + } + args = append(args, "up", "-d") + if res := r.Run(ctx, "docker", args...); res.Err != nil { + return fmt.Errorf("start host services: %w", res.Err) + } + return nil +} + +func InstallCenter(ctx context.Context, r Runner, plan CenterInstallPlan) error { + if res := r.Run(ctx, "mkdir", "-p", plan.WorkDir); res.Err != nil { + return fmt.Errorf("create center install dir: %w", res.Err) + } + if err := loadImages(ctx, r, plan.Images); err != nil { + return err + } + args := []string{"compose", "-f", plan.ComposeFile} + if plan.EnvFile != "" { + args = append(args, "--env-file", plan.EnvFile) + } + args = append(args, "up", "-d") + if res := r.Run(ctx, "docker", args...); res.Err != nil { + return fmt.Errorf("start center services: %w", res.Err) + } + return nil +} + +func PrepareCenterFiles(ctx context.Context, r Runner, plan CenterFilesPlan) error { + if err := copyFile(filepath.Join(plan.PackageDir, "docker-compose.yml"), filepath.Join(plan.WorkDir, "docker-compose.yml")); err != nil { + return err + } + if err := copyTree(filepath.Join(plan.PackageDir, "static"), filepath.Join(plan.WorkDir, "static")); err != nil { + return err + } + if err := copyTree(filepath.Join(plan.PackageDir, "images"), filepath.Join(plan.WorkDir, "images")); err != nil { + return err + } + env, err := os.ReadFile(filepath.Join(plan.PackageDir, ".env.example")) + if err != nil { + return err + } + content := RenderCenterEnv(string(env), plan.Env) + return writeFileAtomic(filepath.Join(plan.WorkDir, ".env"), []byte(content), 0600) +} + +func NewCenterEnv(input CenterEnvInput) (CenterEnv, error) { + env := CenterEnv{ + AccessHost: input.AccessHost, + NginxPort: fallback(input.NginxPort, "80"), + PostgresDB: "monkeycode-ai", + PostgresUser: "monkeycode-ai", + ClickHouseDB: "monkeycode-ai", + ClickHouseUser: "monkeycode-ai", + TeamEmail: input.TeamEmail, + TeamName: fallback(input.TeamName, "MonkeyCode"), + TeamPassword: input.TeamPassword, + SubnetPrefix: "10.100.50", + } + var err error + if env.TeamPassword == "" { + env.TeamPassword, err = randomSecret(24) + if err != nil { + return CenterEnv{}, err + } + } + if env.PostgresPassword, err = randomSecret(24); err != nil { + return CenterEnv{}, err + } + if env.RedisPassword, err = randomSecret(24); err != nil { + return CenterEnv{}, err + } + if env.ClickHousePassword, err = randomSecret(24); err != nil { + return CenterEnv{}, err + } + if env.RustFSAccessKey, err = randomSecret(24); err != nil { + return CenterEnv{}, err + } + if env.RustFSSecretKey, err = randomSecret(32); err != nil { + return CenterEnv{}, err + } + return env, nil +} + +func RenderCenterEnv(template string, env CenterEnv) string { + values := map[string]string{ + "REMOTE_IP": env.AccessHost, + "NGINX_PORT": env.NginxPort, + "POSTGRES_DB": env.PostgresDB, + "POSTGRES_USER": env.PostgresUser, + "POSTGRES_PASSWORD": env.PostgresPassword, + "REDIS_PASSWORD": env.RedisPassword, + "CLICKHOUSE_DB": env.ClickHouseDB, + "CLICKHOUSE_USER": env.ClickHouseUser, + "CLICKHOUSE_PASSWORD": env.ClickHousePassword, + "RUSTFS_ACCESS_KEY": env.RustFSAccessKey, + "RUSTFS_SECRET_KEY": env.RustFSSecretKey, + "TEAM_EMAIL": env.TeamEmail, + "TEAM_NAME": env.TeamName, + "TEAM_PASSWORD": env.TeamPassword, + "SUBNET_PREFIX": env.SubnetPrefix, + } + lines := strings.Split(template, "\n") + for i, line := range lines { + key, _, ok := strings.Cut(line, "=") + if !ok { + continue + } + if value, exists := values[key]; exists { + lines[i] = key + "=" + value + } + } + return strings.Join(lines, "\n") +} + +func loadImages(ctx context.Context, r Runner, images []ImageArchive) error { + for _, image := range images { + if image.Compressed { + if res := r.RunShell(ctx, fmt.Sprintf("gzip -dc '%s' | docker load", shellQuote(image.Path))); res.Err != nil { + return fmt.Errorf("load image %s: %w", image.Path, res.Err) + } + continue + } + if res := r.Run(ctx, "docker", "load", "-i", image.Path); res.Err != nil { + return fmt.Errorf("load image %s: %w", image.Path, res.Err) + } + } + return nil +} + +func UninstallHost(ctx context.Context, r Runner, plan HostInstallPlan) error { + args := []string{"compose", "-f", plan.ComposeFile} + if plan.EnvFile != "" { + args = append(args, "--env-file", plan.EnvFile) + } + args = append(args, "down") + if res := r.Run(ctx, "docker", args...); res.Err != nil { + return fmt.Errorf("stop host services: %w", res.Err) + } + if res := r.Run(ctx, "rm", "-rf", plan.WorkDir); res.Err != nil { + return fmt.Errorf("remove host install dir: %w", res.Err) + } + return nil +} + +func shellJoin(values []string) string { + parts := make([]string, 0, len(values)) + for _, value := range values { + parts = append(parts, "'"+shellQuote(value)+"'") + } + return strings.Join(parts, " ") +} + +func copyTree(src, dst string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if d.IsDir() { + return os.MkdirAll(target, 0755) + } + if shouldSkipPackageFile(filepath.Base(path)) { + return nil + } + return copyFile(path, target) + }) +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, in) + return err +} + +func writeFileAtomic(path string, data []byte, perm os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, perm); err != nil { + return err + } + return os.Rename(tmp, path) +} + +func shouldSkipPackageFile(name string) bool { + return strings.HasPrefix(name, "._") || name == ".DS_Store" +} + +func fallback(value, defaultValue string) string { + if value != "" { + return value + } + return defaultValue +} + +func randomSecret(length int) (string, error) { + raw := make([]byte, length) + if _, err := rand.Read(raw); err != nil { + return "", err + } + secret := base64.RawURLEncoding.EncodeToString(raw) + if len(secret) > length { + return secret[:length], nil + } + return secret, nil +} + +func shellQuote(s string) string { + return strings.ReplaceAll(s, "'", `'\''`) +} + +var ( + dockerVersionRe = regexp.MustCompile(`Docker version ([^,\s]+)`) + composeVersionRe = regexp.MustCompile(`Docker Compose version ([^\s]+)`) +) + +func parseDockerVersion(out string) string { + m := dockerVersionRe.FindStringSubmatch(out) + if len(m) == 2 { + return m[1] + } + return strings.TrimSpace(out) +} + +func parseComposeVersion(out string) string { + m := composeVersionRe.FindStringSubmatch(out) + if len(m) == 2 { + return m[1] + } + return strings.TrimSpace(out) +} diff --git a/backend/pkg/installer/installer_test.go b/backend/pkg/installer/installer_test.go new file mode 100644 index 00000000..2f91b66c --- /dev/null +++ b/backend/pkg/installer/installer_test.go @@ -0,0 +1,408 @@ +package installer + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +type fakeRunner struct { + outputs map[string]RunResult + calls []string +} + +func (r *fakeRunner) Run(ctx context.Context, name string, args ...string) RunResult { + cmd := strings.Join(append([]string{name}, args...), " ") + r.calls = append(r.calls, cmd) + if out, ok := r.outputs[cmd]; ok { + return out + } + return RunResult{Err: errNotFound} +} + +func (r *fakeRunner) RunShell(ctx context.Context, script string) RunResult { + r.calls = append(r.calls, script) + if out, ok := r.outputs[script]; ok { + return out + } + return RunResult{Err: errNotFound} +} + +func TestCheckDockerStatusParsesVersions(t *testing.T) { + r := &fakeRunner{outputs: map[string]RunResult{ + "docker --version": {Stdout: "Docker version 29.4.3, build 123456\n"}, + "docker compose version": {Stdout: "Docker Compose version v2.40.3\n"}, + "docker info --format {{.ServerVersion}}": {Stdout: "29.4.3\n"}, + }} + + status := CheckDockerStatus(context.Background(), r) + + if !status.DockerInstalled { + t.Fatal("DockerInstalled = false") + } + if status.DockerVersion != "29.4.3" { + t.Fatalf("DockerVersion = %q", status.DockerVersion) + } + if !status.ComposeInstalled { + t.Fatal("ComposeInstalled = false") + } + if status.ComposeVersion != "v2.40.3" { + t.Fatalf("ComposeVersion = %q", status.ComposeVersion) + } + if !status.DaemonRunning { + t.Fatal("DaemonRunning = false") + } +} + +func TestCheckDockerStatusDetectsMissingCompose(t *testing.T) { + r := &fakeRunner{outputs: map[string]RunResult{ + "docker --version": {Stdout: "Docker version 29.4.3, build 123456\n"}, + }} + + status := CheckDockerStatus(context.Background(), r) + + if !status.DockerInstalled { + t.Fatal("DockerInstalled = false") + } + if status.ComposeInstalled { + t.Fatal("ComposeInstalled = true") + } + if status.Ready() { + t.Fatal("Ready() = true") + } +} + +func TestInstallHostLoadsImagesAndStartsCompose(t *testing.T) { + r := &fakeRunner{outputs: map[string]RunResult{ + "docker load -i /opt/monkeycode-host/images/orchestrator.tar": {}, + "gzip -dc '/opt/monkeycode-host/images/orchestrator.tgz' | docker load": {}, + "sh -c printf '%s\\n' 'TOKEN=host-token' 'GRPC_URL=10.0.0.1:50443' >> '/opt/monkeycode-host/.env'": {}, + "docker compose -f /opt/monkeycode-host/docker-compose.yml --env-file /opt/monkeycode-host/.env up -d": {}, + }} + plan := HostInstallPlan{ + WorkDir: "/opt/monkeycode-host", + ComposeFile: "/opt/monkeycode-host/docker-compose.yml", + EnvFile: "/opt/monkeycode-host/.env", + Token: "host-token", + GrpcURL: "10.0.0.1:50443", + Images: []ImageArchive{ + {Path: "/opt/monkeycode-host/images/orchestrator.tar", Compressed: false}, + {Path: "/opt/monkeycode-host/images/orchestrator.tgz", Compressed: true}, + }, + } + + if err := InstallHost(context.Background(), r, plan); err != nil { + t.Fatalf("%v, calls=%#v", err, r.calls) + } + + want := []string{ + "docker load -i /opt/monkeycode-host/images/orchestrator.tar", + "gzip -dc '/opt/monkeycode-host/images/orchestrator.tgz' | docker load", + "sh -c printf '%s\\n' 'TOKEN=host-token' 'GRPC_URL=10.0.0.1:50443' >> '/opt/monkeycode-host/.env'", + "docker compose -f /opt/monkeycode-host/docker-compose.yml --env-file /opt/monkeycode-host/.env up -d", + } + if strings.Join(r.calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls = %#v", r.calls) + } +} + +func TestUninstallHostStopsComposeAndRemovesWorkDir(t *testing.T) { + r := &fakeRunner{outputs: map[string]RunResult{ + "docker compose -f /opt/monkeycode-host/docker-compose.yml --env-file /opt/monkeycode-host/.env down": {}, + "rm -rf /opt/monkeycode-host": {}, + }} + plan := HostInstallPlan{ + WorkDir: "/opt/monkeycode-host", + ComposeFile: "/opt/monkeycode-host/docker-compose.yml", + EnvFile: "/opt/monkeycode-host/.env", + } + + if err := UninstallHost(context.Background(), r, plan); err != nil { + t.Fatalf("%v, calls=%#v", err, r.calls) + } + + want := []string{ + "docker compose -f /opt/monkeycode-host/docker-compose.yml --env-file /opt/monkeycode-host/.env down", + "rm -rf /opt/monkeycode-host", + } + if strings.Join(r.calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls = %#v", r.calls) + } +} + +func TestInstallCenterLoadsImagesAndStartsCompose(t *testing.T) { + r := &fakeRunner{outputs: map[string]RunResult{ + "mkdir -p /data/monkeycode-ai": {}, + "docker load -i /data/monkeycode-ai/images/backend.tar": {}, + "gzip -dc '/data/monkeycode-ai/images/frontend.tar.gz' | docker load": {}, + "docker compose -f /data/monkeycode-ai/docker-compose.yml --env-file /data/monkeycode-ai/.env up -d": {}, + }} + plan := CenterInstallPlan{ + WorkDir: "/data/monkeycode-ai", + ComposeFile: "/data/monkeycode-ai/docker-compose.yml", + EnvFile: "/data/monkeycode-ai/.env", + Images: []ImageArchive{ + {Path: "/data/monkeycode-ai/images/backend.tar"}, + {Path: "/data/monkeycode-ai/images/frontend.tar.gz", Compressed: true}, + }, + } + + if err := InstallCenter(context.Background(), r, plan); err != nil { + t.Fatalf("%v, calls=%#v", err, r.calls) + } + + want := []string{ + "mkdir -p /data/monkeycode-ai", + "docker load -i /data/monkeycode-ai/images/backend.tar", + "gzip -dc '/data/monkeycode-ai/images/frontend.tar.gz' | docker load", + "docker compose -f /data/monkeycode-ai/docker-compose.yml --env-file /data/monkeycode-ai/.env up -d", + } + if strings.Join(r.calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls = %#v", r.calls) + } +} + +func TestInstallDockerFromLocalBundleDoesNotDownload(t *testing.T) { + r := &fakeRunner{outputs: map[string]RunResult{ + "mkdir -p /tmp/monkeycode-installer": {}, + "tar -zxf /pkg/docker.tgz -C /tmp/monkeycode-installer": {}, + "install -m 0755 '/tmp/monkeycode-installer'/docker/* /usr/local/bin/": {}, + "install -m 0755 /tmp/monkeycode-installer/docker-compose /usr/local/lib/docker/cli-plugins/docker-compose": {}, + "ln -sf /usr/local/lib/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose": {}, + "systemctl daemon-reload": {}, + "systemctl enable --now docker": {}, + }} + + err := InstallDockerFromLocalBundle(context.Background(), r, DockerInstallPlan{ + WorkDir: "/tmp/monkeycode-installer", + BundleFile: "/pkg/docker.tgz", + }) + if err != nil { + t.Fatal(err) + } +} + +func TestPrepareCenterFilesCopiesPackageFilesAndWritesEnv(t *testing.T) { + pkg := t.TempDir() + work := filepath.Join(t.TempDir(), "monkeycode-ai") + writeFile(t, filepath.Join(pkg, "docker-compose.yml"), "compose") + writeFile(t, filepath.Join(pkg, ".env.example"), "REMOTE_IP=\nNGINX_PORT=\nTEAM_EMAIL=\nTEAM_NAME=\nTEAM_PASSWORD=\nPOSTGRES_PASSWORD=\n") + writeFile(t, filepath.Join(pkg, "static", "project-tpl.zip"), "zip") + writeFile(t, filepath.Join(pkg, "images", "backend.tar.gz"), "image") + + err := PrepareCenterFiles(context.Background(), CommandRunner{}, CenterFilesPlan{ + PackageDir: pkg, + WorkDir: work, + Env: CenterEnv{ + AccessHost: "192.168.1.10", + NginxPort: "8080", + TeamEmail: "admin@example.com", + TeamName: "Example", + TeamPassword: "team-secret", + PostgresPassword: "pg-secret", + }, + }) + if err != nil { + t.Fatal(err) + } + + if got := readFile(t, filepath.Join(work, "docker-compose.yml")); got != "compose" { + t.Fatalf("compose = %q", got) + } + if got := readFile(t, filepath.Join(work, "static", "project-tpl.zip")); got != "zip" { + t.Fatalf("project-tpl.zip = %q", got) + } + env := readFile(t, filepath.Join(work, ".env")) + if !strings.Contains(env, "REMOTE_IP=192.168.1.10") { + t.Fatalf("env = %q", env) + } + for _, want := range []string{ + "NGINX_PORT=8080", + "TEAM_EMAIL=admin@example.com", + "TEAM_NAME=Example", + "TEAM_PASSWORD=team-secret", + "POSTGRES_PASSWORD=pg-secret", + } { + if !strings.Contains(env, want) { + t.Fatalf("env missing %q: %q", want, env) + } + } +} + +func TestNewCenterEnvGeneratesPasswords(t *testing.T) { + env, err := NewCenterEnv(CenterEnvInput{ + AccessHost: "192.168.1.10", + NginxPort: "80", + TeamEmail: "admin@example.com", + TeamName: "MonkeyCode", + }) + if err != nil { + t.Fatal(err) + } + for name, value := range map[string]string{ + "TeamPassword": env.TeamPassword, + "PostgresPassword": env.PostgresPassword, + "RedisPassword": env.RedisPassword, + "ClickHousePassword": env.ClickHousePassword, + "RustFSAccessKey": env.RustFSAccessKey, + "RustFSSecretKey": env.RustFSSecretKey, + } { + if len(value) < 24 { + t.Fatalf("%s too short: %q", name, value) + } + } + if env.NginxPort != "80" { + t.Fatalf("NginxPort = %q", env.NginxPort) + } +} + +func TestRenderCenterEnvReplacesKeys(t *testing.T) { + got := RenderCenterEnv("REMOTE_IP=\nNGINX_PORT=\nKEEP=value\n", CenterEnv{ + AccessHost: "example.com", + NginxPort: "8080", + }) + if !strings.Contains(got, "REMOTE_IP=example.com") { + t.Fatalf("env = %q", got) + } + if !strings.Contains(got, "NGINX_PORT=8080") { + t.Fatalf("env = %q", got) + } + if !strings.Contains(got, "KEEP=value") { + t.Fatalf("env = %q", got) + } +} + +func TestInstallDockerDownloadsBundleAndInstallsBinaries(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("docker bundle")) + })) + defer server.Close() + + workDir := t.TempDir() + bundleFile := filepath.Join(workDir, "docker.tgz") + r := &fakeRunner{outputs: map[string]RunResult{ + "mkdir -p " + workDir: {}, + "tar -zxf " + bundleFile + " -C " + workDir: {}, + "install -m 0755 '" + workDir + "'/docker/* /usr/local/bin/": {}, + "install -m 0755 " + workDir + "/docker-compose /usr/local/lib/docker/cli-plugins/docker-compose": {}, + "ln -sf /usr/local/lib/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose": {}, + "systemctl daemon-reload": {}, + "systemctl enable --now docker": {}, + }} + plan := DockerInstallPlan{ + BundleURL: server.URL, + WorkDir: workDir, + BundleFile: bundleFile, + } + + if err := InstallDocker(context.Background(), r, plan); err != nil { + t.Fatal(err) + } + + want := []string{ + "mkdir -p " + workDir, + "tar -zxf " + bundleFile + " -C " + workDir, + "install -m 0755 '" + workDir + "'/docker/* /usr/local/bin/", + "install -m 0755 " + workDir + "/docker-compose /usr/local/lib/docker/cli-plugins/docker-compose", + "ln -sf /usr/local/lib/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose", + "systemctl daemon-reload", + "systemctl enable --now docker", + } + if strings.Join(r.calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls = %#v", r.calls) + } +} + +func TestPrepareHostDownloadsBundle(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("host bundle")) + })) + defer server.Close() + + r := &fakeRunner{outputs: map[string]RunResult{ + "mkdir -p /opt/monkeycode-host": {}, + "tar -zxf /tmp/monkeycode-host.tgz -C /opt/monkeycode-host": {}, + }} + plan := HostBundlePlan{ + BundleURL: server.URL, + BundleFile: "/tmp/monkeycode-host.tgz", + WorkDir: "/opt/monkeycode-host", + } + + if err := PrepareHostBundle(context.Background(), r, plan); err != nil { + t.Fatal(err) + } + + want := []string{ + "mkdir -p /opt/monkeycode-host", + "tar -zxf /tmp/monkeycode-host.tgz -C /opt/monkeycode-host", + } + if strings.Join(r.calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls = %#v", r.calls) + } +} + +func TestDownloadFileReportsProgress(t *testing.T) { + body := strings.Repeat("a", 1024) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "1024") + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + + var events []DownloadProgress + dest := filepath.Join(t.TempDir(), "bundle.tgz") + if err := DownloadFile(context.Background(), server.URL, dest, func(progress DownloadProgress) { + events = append(events, progress) + }); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(dest) + if err != nil { + t.Fatal(err) + } + if string(got) != body { + t.Fatalf("downloaded body = %q", string(got)) + } + if len(events) == 0 { + t.Fatal("progress callback was not called") + } + last := events[len(events)-1] + if last.Downloaded != 1024 || last.Total != 1024 || last.Percent() != 1 { + t.Fatalf("last progress = %#v percent=%v", last, last.Percent()) + } +} + +func TestBundleURLJoinsBaseURLAndPath(t *testing.T) { + url, err := BundleURL("http://server/base/", "/static/host/installation.tgz") + if err != nil { + t.Fatal(err) + } + if url != "http://server/base/static/host/installation.tgz" { + t.Fatalf("url = %q", url) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} diff --git a/backend/pkg/installer/tls.go b/backend/pkg/installer/tls.go new file mode 100644 index 00000000..667a9dc6 --- /dev/null +++ b/backend/pkg/installer/tls.go @@ -0,0 +1,66 @@ +package installer + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +type TLSPlan struct { + Host string + CertFile string + KeyFile string +} + +func GenerateSelfSignedTLS(plan TLSPlan) error { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return err + } + + cert := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: plan.Host, + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().AddDate(10, 0, 0), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + if ip := net.ParseIP(plan.Host); ip != nil { + cert.IPAddresses = []net.IP{ip} + } else { + cert.DNSNames = []string{plan.Host} + } + + der, err := x509.CreateCertificate(rand.Reader, cert, cert, &key.PublicKey, key) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(plan.CertFile), 0755); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(plan.KeyFile), 0755); err != nil { + return err + } + if err := os.WriteFile(plan.CertFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0644); err != nil { + return err + } + + keyBytes := x509.MarshalPKCS1PrivateKey(key) + return os.WriteFile(plan.KeyFile, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: keyBytes}), 0600) +} diff --git a/backend/pkg/installer/tls_test.go b/backend/pkg/installer/tls_test.go new file mode 100644 index 00000000..bb1e4e02 --- /dev/null +++ b/backend/pkg/installer/tls_test.go @@ -0,0 +1,69 @@ +package installer + +import ( + "crypto/x509" + "encoding/pem" + "net" + "os" + "path/filepath" + "testing" +) + +func TestGenerateSelfSignedTLSWithIPSAN(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + + err := GenerateSelfSignedTLS(TLSPlan{ + Host: "192.168.1.10", + CertFile: certFile, + KeyFile: keyFile, + }) + if err != nil { + t.Fatal(err) + } + + cert := readCert(t, certFile) + if len(cert.IPAddresses) != 1 || !cert.IPAddresses[0].Equal(net.ParseIP("192.168.1.10")) { + t.Fatalf("IPAddresses = %#v", cert.IPAddresses) + } + if _, err := os.Stat(keyFile); err != nil { + t.Fatal(err) + } +} + +func TestGenerateSelfSignedTLSWithDNSSAN(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "server.crt") + + err := GenerateSelfSignedTLS(TLSPlan{ + Host: "monkeycode.local", + CertFile: certFile, + KeyFile: filepath.Join(dir, "server.key"), + }) + if err != nil { + t.Fatal(err) + } + + cert := readCert(t, certFile) + if len(cert.DNSNames) != 1 || cert.DNSNames[0] != "monkeycode.local" { + t.Fatalf("DNSNames = %#v", cert.DNSNames) + } +} + +func readCert(t *testing.T, certFile string) *x509.Certificate { + t.Helper() + data, err := os.ReadFile(certFile) + if err != nil { + t.Fatal(err) + } + block, _ := pem.Decode(data) + if block == nil { + t.Fatal("missing PEM block") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatal(err) + } + return cert +} diff --git a/backend/pkg/oss/oss.go b/backend/pkg/oss/oss.go new file mode 100644 index 00000000..12dec82a --- /dev/null +++ b/backend/pkg/oss/oss.go @@ -0,0 +1,343 @@ +package oss + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/chaitin/MonkeyCode/backend/config" +) + +const ( + defaultExpires = 10 * time.Minute + maxExpires = 7 * 24 * time.Hour +) + +type S3Option struct { + ForcePathStyle bool + InitBucket bool +} + +type Client struct { + cfg config.ObjectStorageConfig + region string + s3 *s3.Client + presigner *s3.PresignClient + pathStyle bool +} + +type Presign struct { + UploadURL string + AccessURL string +} + +func NewS3Compatible(ctx context.Context, cfg config.ObjectStorageConfig, opt S3Option) (*Client, error) { + if err := validateConfig(cfg); err != nil { + return nil, err + } + region := normalizeRegion(cfg.Region) + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.AccessKeySecret, "")), + ) + if err != nil { + return nil, err + } + client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(cfg.Endpoint) + o.UsePathStyle = opt.ForcePathStyle + }) + c := &Client{ + cfg: cfg, + region: region, + s3: client, + pathStyle: opt.ForcePathStyle, + presigner: s3.NewPresignClient(client, s3.WithPresignClientFromClientOptions(func(o *s3.Options) { + o.BaseEndpoint = aws.String(presignSigningEndpoint(cfg)) + o.UsePathStyle = opt.ForcePathStyle + })), + } + if opt.InitBucket { + if err := c.initBucket(ctx); err != nil { + return nil, err + } + } + return c, nil +} + +func validateConfig(cfg config.ObjectStorageConfig) error { + if strings.TrimSpace(cfg.Endpoint) == "" { + return errors.New("oss endpoint is empty") + } + if strings.TrimSpace(cfg.AccessKey) == "" { + return errors.New("oss access key is empty") + } + if strings.TrimSpace(cfg.AccessKeySecret) == "" { + return errors.New("oss access key secret is empty") + } + if strings.TrimSpace(cfg.Bucket) == "" { + return errors.New("oss bucket is empty") + } + return nil +} + +func (c *Client) initBucket(ctx context.Context) error { + input := &s3.CreateBucketInput{ + Bucket: aws.String(c.cfg.Bucket), + } + if c.region != "" && c.region != "us-east-1" { + input.CreateBucketConfiguration = &s3types.CreateBucketConfiguration{ + LocationConstraint: s3types.BucketLocationConstraint(c.region), + } + } + _, err := c.s3.CreateBucket(ctx, input) + if err != nil { + var bucketOwned *s3types.BucketAlreadyOwnedByYou + if !errors.As(err, &bucketOwned) { + return err + } + } + return c.initPublicReadPolicy(ctx) +} + +func (c *Client) initPublicReadPolicy(ctx context.Context) error { + resources := publicReadResources(c.cfg) + if len(resources) == 0 { + return nil + } + policy := bucketPolicy{ + Version: "2012-10-17", + Statement: []bucketPolicyStatement{ + { + Effect: "Allow", + Principal: bucketPolicyPrincipal{AWS: []string{"*"}}, + Action: []string{"s3:GetObject"}, + Resource: resources, + }, + }, + } + data, err := json.Marshal(policy) + if err != nil { + return err + } + _, err = c.s3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ + Bucket: aws.String(c.cfg.Bucket), + Policy: aws.String(string(data)), + }) + return err +} + +func (c *Client) PutFile(ctx context.Context, prefix, filename string, r io.Reader) error { + _, err := c.s3.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.cfg.Bucket), + Key: aws.String(objectKey(prefix, filename)), + Body: r, + }) + return err +} + +func (c *Client) HeadFile(ctx context.Context, prefix, filename string) (bool, error) { + _, err := c.s3.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(c.cfg.Bucket), + Key: aws.String(objectKey(prefix, filename)), + }) + if err == nil { + return true, nil + } + var notFound *s3types.NotFound + if errors.As(err, ¬Found) { + return false, nil + } + return false, err +} + +func (c *Client) WithAccessEndpoint(endpoint string) *Client { + endpoint = strings.TrimSpace(endpoint) + if c == nil || endpoint == "" { + return c + } + next := *c + next.cfg.AccessEndpoint = endpoint + if c.s3 != nil { + next.presigner = s3.NewPresignClient(c.s3, s3.WithPresignClientFromClientOptions(func(o *s3.Options) { + o.BaseEndpoint = aws.String(presignSigningEndpoint(next.cfg)) + o.UsePathStyle = c.pathStyle + })) + } + return &next +} + +func (c *Client) GetURL(prefix, filename string) string { + base := objectAccessBase(c.cfg) + return appendURLPath(base, objectKey(prefix, filename)) +} + +func (c *Client) Presign(ctx context.Context, prefix, filename string, expires time.Duration) (*Presign, error) { + expires = normalizeExpires(expires) + key := objectKey(prefix, filename) + putURL, err := c.presigner.PresignPutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.cfg.Bucket), + Key: aws.String(key), + }, s3.WithPresignExpires(expires)) + if err != nil { + return nil, fmt.Errorf("presign put object: %w", err) + } + getURL, err := c.presigner.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(c.cfg.Bucket), + Key: aws.String(key), + }, s3.WithPresignExpires(expires)) + if err != nil { + return nil, fmt.Errorf("presign get object: %w", err) + } + return &Presign{ + UploadURL: c.publicPresignURL(putURL.URL, key), + AccessURL: c.publicPresignURL(getURL.URL, key), + }, nil +} + +func objectKey(prefix, filename string) string { + cleanPrefix := cleanObjectPrefix(prefix) + name := path.Base(strings.Trim(filename, "/")) + if name == "." || name == ".." || name == "/" { + name = "" + } + return strings.Trim(path.Join(cleanPrefix, name), "/") +} + +func normalizeExpires(expires time.Duration) time.Duration { + if expires <= 0 { + return defaultExpires + } + if expires > maxExpires { + return maxExpires + } + return expires +} + +func normalizeRegion(region string) string { + region = strings.TrimSpace(region) + if region == "" { + return "us-east-1" + } + return region +} + +func cleanObjectPrefix(prefix string) string { + cleanPrefix := strings.Trim(path.Clean(strings.Trim(prefix, "/")), "/") + for cleanPrefix == ".." || strings.HasPrefix(cleanPrefix, "../") { + cleanPrefix = strings.TrimPrefix(cleanPrefix, "../") + if cleanPrefix == ".." { + return "" + } + } + if cleanPrefix == "." { + return "" + } + return cleanPrefix +} + +type bucketPolicy struct { + Version string `json:"Version"` + Statement []bucketPolicyStatement `json:"Statement"` +} + +type bucketPolicyStatement struct { + Effect string `json:"Effect"` + Principal bucketPolicyPrincipal `json:"Principal"` + Action []string `json:"Action"` + Resource []string `json:"Resource"` +} + +type bucketPolicyPrincipal struct { + AWS []string `json:"AWS"` +} + +func publicReadResources(cfg config.ObjectStorageConfig) []string { + prefixes := []string{cfg.AvatarPrefix, cfg.SpecPrefix, cfg.RepoPrefix} + resources := make([]string, 0, len(prefixes)) + seen := make(map[string]struct{}, len(prefixes)) + bucket := strings.Trim(cfg.Bucket, "/") + for _, prefix := range prefixes { + prefix = cleanObjectPrefix(prefix) + if prefix == "" { + continue + } + if _, ok := seen[prefix]; ok { + continue + } + seen[prefix] = struct{}{} + resources = append(resources, fmt.Sprintf("arn:aws:s3:::%s/%s/*", bucket, prefix)) + } + return resources +} + +func appendURLPath(base, key string) string { + u, err := url.Parse(base) + if err != nil { + return strings.TrimRight(base, "/") + "/" + key + } + u.Path = strings.TrimRight(u.Path, "/") + "/" + key + return u.String() +} + +func objectAccessBase(cfg config.ObjectStorageConfig) string { + base := strings.TrimRight(strings.TrimSpace(cfg.AccessEndpoint), "/") + if base == "" { + base = strings.TrimRight(cfg.Endpoint, "/") + } + bucket := strings.Trim(cfg.Bucket, "/") + if bucket == "" { + return base + } + u, err := url.Parse(base) + if err != nil { + return strings.TrimRight(base, "/") + "/" + bucket + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) == 0 || parts[len(parts)-1] != bucket { + u.Path = strings.TrimRight(u.Path, "/") + "/" + bucket + } + return u.String() +} + +func presignSigningEndpoint(cfg config.ObjectStorageConfig) string { + endpoint := strings.TrimSpace(cfg.AccessEndpoint) + if endpoint == "" { + return cfg.Endpoint + } + u, err := url.Parse(endpoint) + if err != nil { + return endpoint + } + u.Path = "" + u.RawPath = "" + u.RawQuery = "" + u.Fragment = "" + return strings.TrimRight(u.String(), "/") +} + +func (c *Client) publicPresignURL(signedURL, key string) string { + publicURL := appendURLPath(objectAccessBase(c.cfg), key) + public, err := url.Parse(publicURL) + if err != nil { + return signedURL + } + signed, err := url.Parse(signedURL) + if err != nil { + return publicURL + } + public.RawQuery = signed.RawQuery + return public.String() +} diff --git a/backend/pkg/oss/oss_test.go b/backend/pkg/oss/oss_test.go new file mode 100644 index 00000000..5e25d285 --- /dev/null +++ b/backend/pkg/oss/oss_test.go @@ -0,0 +1,341 @@ +package oss + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/chaitin/MonkeyCode/backend/config" +) + +func TestObjectKeyJoinsPrefixAndFilename(t *testing.T) { + key := objectKey("/tmp/task-attachments/", "/a.txt") + if key != "tmp/task-attachments/a.txt" { + t.Fatalf("key = %q", key) + } +} + +func TestObjectKeyKeepsFilenameInsidePrefix(t *testing.T) { + key := objectKey("tmp/task-attachments", "../a #?.txt") + if key != "tmp/task-attachments/a #?.txt" { + t.Fatalf("key = %q", key) + } +} + +func TestNormalizeExpires(t *testing.T) { + if got := normalizeExpires(0); got != 10*time.Minute { + t.Fatalf("zero expires = %s", got) + } + if got := normalizeExpires(8 * 24 * time.Hour); got != 7*24*time.Hour { + t.Fatalf("large expires = %s", got) + } +} + +func TestPublicURLUsesAccessEndpoint(t *testing.T) { + client := &Client{ + cfg: config.ObjectStorageConfig{ + AccessEndpoint: "http://localhost:9000/monkeycode-private", + }, + } + url := client.GetURL("tmp/task-attachments", "a.txt") + if url != "http://localhost:9000/monkeycode-private/tmp/task-attachments/a.txt" { + t.Fatalf("url = %q", url) + } +} + +func TestPublicURLAddsBucketWhenAccessEndpointHasNoPath(t *testing.T) { + client := &Client{ + cfg: config.ObjectStorageConfig{ + AccessEndpoint: "http://localhost:9000", + Bucket: "monkeycode-private", + }, + } + url := client.GetURL("tmp/task-attachments", "a.txt") + if url != "http://localhost:9000/monkeycode-private/tmp/task-attachments/a.txt" { + t.Fatalf("url = %q", url) + } +} + +func TestPublicURLEscapesPath(t *testing.T) { + client := &Client{ + cfg: config.ObjectStorageConfig{ + AccessEndpoint: "http://localhost:9000/monkeycode-private", + }, + } + url := client.GetURL("tmp", "a #?.txt") + if url != "http://localhost:9000/monkeycode-private/tmp/a%20%23%3F.txt" { + t.Fatalf("url = %q", url) + } +} + +func TestValidateConfigRequiresS3Fields(t *testing.T) { + err := validateConfig(config.ObjectStorageConfig{}) + if err == nil { + t.Fatal("expected config error") + } +} + +func TestHeadFileReturnsTrueWhenObjectExists(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + t.Fatalf("method = %s, want HEAD", r.Method) + } + if r.URL.Path != "/bucket/repo/project-tpl.zip" { + t.Fatalf("path = %s", r.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: server.URL, + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "bucket", + }, S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + exists, err := client.HeadFile(context.Background(), "repo", "project-tpl.zip") + if err != nil { + t.Fatal(err) + } + if !exists { + t.Fatal("exists = false, want true") + } +} + +func TestHeadFileReturnsFalseWhenObjectMissing(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`NoSuchKeymissing`)) + })) + defer server.Close() + + client, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: server.URL, + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "bucket", + }, S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + exists, err := client.HeadFile(context.Background(), "repo", "project-tpl.zip") + if err != nil { + t.Fatal(err) + } + if exists { + t.Fatal("exists = true, want false") + } +} + +func TestPresignUsesAccessEndpointHost(t *testing.T) { + client, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: "http://internal:9000", + AccessEndpoint: "http://public.example.com/monkeycode-private", + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "monkeycode-private", + }, S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + presign, err := client.Presign(context.Background(), "tmp", "a.txt", time.Minute) + if err != nil { + t.Fatal(err) + } + if strings.Contains(presign.UploadURL, "internal:9000") || strings.Contains(presign.AccessURL, "internal:9000") { + t.Fatalf("presign url uses internal endpoint: %#v", presign) + } + if !strings.Contains(presign.UploadURL, "public.example.com") || !strings.Contains(presign.AccessURL, "public.example.com") { + t.Fatalf("presign url does not use access endpoint: %#v", presign) + } +} + +func TestPresignWithAccessEndpointOverridesConfiguredEndpoint(t *testing.T) { + client, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: "http://internal:9000", + AccessEndpoint: "http://old.example.com/monkeycode-private", + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "monkeycode-private", + }, S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + presign, err := client.WithAccessEndpoint("https://new.example.com").Presign(context.Background(), "tmp", "a.txt", time.Minute) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(presign.UploadURL, "new.example.com") || !strings.Contains(presign.AccessURL, "new.example.com") { + t.Fatalf("presign url does not use request access endpoint: %#v", presign) + } + if strings.Contains(presign.UploadURL, "old.example.com") || strings.Contains(presign.AccessURL, "old.example.com") { + t.Fatalf("presign url uses configured endpoint: %#v", presign) + } +} + +func TestPresignWithAccessEndpointKeepsPathPrefixOutsideSignature(t *testing.T) { + client, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: "http://internal:9000", + AccessEndpoint: "https://monkeycode.example.com/oss", + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "monkeycode-private", + }, S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + presign, err := client.Presign(context.Background(), "tmp", "a.txt", time.Minute) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(presign.UploadURL, "/oss/monkeycode-private/tmp/a.txt") { + t.Fatalf("presign upload url path missing /oss prefix: %s", presign.UploadURL) + } + if !strings.Contains(presign.AccessURL, "/oss/monkeycode-private/tmp/a.txt") { + t.Fatalf("presign access url path missing /oss prefix: %s", presign.AccessURL) + } + signingClient, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: "http://internal:9000", + AccessEndpoint: "https://monkeycode.example.com", + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "monkeycode-private", + }, S3Option{ForcePathStyle: true}) + if err != nil { + t.Fatal(err) + } + signingPresign, err := signingClient.Presign(context.Background(), "tmp", "a.txt", time.Minute) + if err != nil { + t.Fatal(err) + } + if signatureValue(t, presign.UploadURL) != signatureValue(t, signingPresign.UploadURL) { + t.Fatalf("presign upload signature includes proxy prefix: %s", presign.UploadURL) + } +} + +func signatureValue(t *testing.T, raw string) string { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u.Query().Get("X-Amz-Signature") +} + +func TestGetURLWithAccessEndpointOverridesConfiguredEndpoint(t *testing.T) { + client := &Client{ + cfg: config.ObjectStorageConfig{ + AccessEndpoint: "http://old.example.com/monkeycode-private", + Bucket: "monkeycode-private", + }, + } + got := client.WithAccessEndpoint("https://new.example.com").GetURL("tmp", "a.txt") + if got != "https://new.example.com/monkeycode-private/tmp/a.txt" { + t.Fatalf("url = %q", got) + } +} + +func TestInitBucketReturnsBucketAlreadyExists(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`BucketAlreadyExistsexists`)) + })) + defer server.Close() + + _, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: server.URL, + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "bucket", + }, S3Option{ForcePathStyle: true, InitBucket: true}) + if err == nil { + t.Fatal("expected bucket already exists error") + } +} + +func TestInitBucketSetsLocationConstraintForRegion(t *testing.T) { + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, _ := io.ReadAll(r.Body) + body = string(data) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + _, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: server.URL, + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "bucket", + Region: "eu-west-1", + }, S3Option{ForcePathStyle: true, InitBucket: true}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(body, "eu-west-1") { + t.Fatalf("create bucket body = %q", body) + } +} + +func TestInitBucketSetsPublicReadPolicyForPermanentPrefixes(t *testing.T) { + var policy string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.RawQuery == "policy=" || r.URL.RawQuery == "policy" { + data, _ := io.ReadAll(r.Body) + policy = string(data) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + _, err := NewS3Compatible(context.Background(), config.ObjectStorageConfig{ + Endpoint: server.URL, + AccessKey: "ak", + AccessKeySecret: "sk", + Bucket: "bucket", + AvatarPrefix: "avatar", + SpecPrefix: "spec", + RepoPrefix: "repo", + TempPrefix: "temp", + }, S3Option{ForcePathStyle: true, InitBucket: true}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"Principal":{"AWS":["*"]}`, + `"arn:aws:s3:::bucket/avatar/*"`, + `"arn:aws:s3:::bucket/spec/*"`, + `"arn:aws:s3:::bucket/repo/*"`, + } { + if !strings.Contains(policy, want) { + t.Fatalf("policy missing %s: %s", want, policy) + } + } + if strings.Contains(policy, "temp") { + t.Fatalf("policy exposes temp prefix: %s", policy) + } +} + +func TestPublicReadResourcesSkipsEmptyAndDuplicatePrefixes(t *testing.T) { + got := publicReadResources(config.ObjectStorageConfig{ + Bucket: "bucket", + AvatarPrefix: "avatar", + SpecPrefix: "/avatar/", + RepoPrefix: "", + TempPrefix: "temp", + }) + if len(got) != 1 { + t.Fatalf("resources = %#v", got) + } + if got[0] != "arn:aws:s3:::bucket/avatar/*" { + t.Fatalf("resource = %q", got[0]) + } +} diff --git a/backend/pkg/register.go b/backend/pkg/register.go index 24b97b73..bcbb7aa4 100644 --- a/backend/pkg/register.go +++ b/backend/pkg/register.go @@ -59,7 +59,8 @@ func RegisterInfra(i *do.Injector, w ...*web.Web) error { do.ProvideValue(i, w[0]) } else { do.Provide(i, func(i *do.Injector) (*web.Web, error) { - return web.New(), nil + w := web.New() + return w, nil }) } diff --git a/backend/pkg/vmstatus/status.go b/backend/pkg/vmstatus/status.go index a107ebd2..154e7553 100644 --- a/backend/pkg/vmstatus/status.go +++ b/backend/pkg/vmstatus/status.go @@ -33,9 +33,7 @@ func Resolve(input Input) taskflow.VirtualMachineStatus { return taskflow.VirtualMachineStatusOffline case etypes.ConditionTypeHibernated: - if last.Reason == "Hibernated" { - return taskflow.VirtualMachineStatusHibernated - } + return taskflow.VirtualMachineStatusHibernated } } diff --git a/backend/scripts/build-offline-package.sh b/backend/scripts/build-offline-package.sh new file mode 100755 index 00000000..ff7435d8 --- /dev/null +++ b/backend/scripts/build-offline-package.sh @@ -0,0 +1,139 @@ +#!/bin/sh +set -eu + +ARCH="${ARCH:-amd64}" +DOCKER_VERSION="${DOCKER_VERSION:-29.0.4}" +PROJECT_TPL_URL="${PROJECT_TPL_URL:-https://baizhiyun.oss-cn-hangzhou.aliyuncs.com/codingmatrix/project-tpl/codingmatrix-project-tpl.master.zip}" +OUT_DIR="${OUT_DIR:-dist/offline}" +PACKAGE_NAME="monkeycode-offline-linux-$ARCH" +PACKAGE_DIR="$OUT_DIR/$PACKAGE_NAME" +GOCACHE="${GOCACHE:-/root/.cache/go-build}" +GOMODCACHE="${GOMODCACHE:-/go/pkg/mod}" +REPO_COMMIT="${REPO_COMMIT:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}" +HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}" +NO_PROXY="${NO_PROXY:-${no_proxy:-}}" +export HTTP_PROXY HTTPS_PROXY NO_PROXY +export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY" + +case "$ARCH" in + amd64) + CENTER_GOARCH="amd64" + CENTER_DOCKER_ARCH="x86_64" + ;; + arm64) + CENTER_GOARCH="arm64" + CENTER_DOCKER_ARCH="aarch64" + ;; + *) + echo "unsupported ARCH=$ARCH" + exit 1 + ;; +esac + +mkdir -p "$OUT_DIR" +rm -rf "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/images" "$PACKAGE_DIR/static/installer/x86_64" "$PACKAGE_DIR/static/installer/aarch64" + +CGO_ENABLED=0 GOOS=linux GOARCH="$CENTER_GOARCH" go build -o "$PACKAGE_DIR/installer" ./cmd/installer +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "$PACKAGE_DIR/static/installer/x86_64/installer" ./cmd/installer +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o "$PACKAGE_DIR/static/installer/aarch64/installer" ./cmd/installer + +curl -fL "https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz" -o "$OUT_DIR/docker-x86_64.tgz" +curl -fL "https://download.docker.com/linux/static/stable/aarch64/docker-$DOCKER_VERSION.tgz" -o "$OUT_DIR/docker-aarch64.tgz" +cp "$OUT_DIR/docker-$CENTER_DOCKER_ARCH.tgz" "$PACKAGE_DIR/docker.tgz" +cp "$OUT_DIR/docker-x86_64.tgz" "$PACKAGE_DIR/static/installer/x86_64/docker.tgz" +cp "$OUT_DIR/docker-aarch64.tgz" "$PACKAGE_DIR/static/installer/aarch64/docker.tgz" + +cp Installation/center/install.sh "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/install.sh" +cp Installation/center/.env.example "$PACKAGE_DIR/.env.example" +sed \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/postgres:17.4-alpine3.21#monkeycode-offline/postgres:local#g' \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/redis:8.0-alpine3.21#monkeycode-offline/redis:local#g' \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/clickhouse-server:26.3.9#monkeycode-offline/clickhouse:local#g' \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/rustfs:1.0.0-beta.2#monkeycode-offline/rustfs:local#g' \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/ingress:[^[:space:]]*#monkeycode-offline/ingress:local#g' \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/taskflow:fc0daba#monkeycode-offline/taskflow:local#g' \ + -e 's#ghcr.1ms.run/chaitin/monkeycode-frontend:[^[:space:]]*#monkeycode-offline/frontend:local#g' \ + -e 's#chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/backend:[^[:space:]]*#monkeycode-offline/backend:local#g' \ + docker-compose.yml > "$PACKAGE_DIR/docker-compose.yml" + +mkdir -p "$PACKAGE_DIR/static" +if [ -d static ]; then + cp -R static/. "$PACKAGE_DIR/static/" +fi +curl -fL "$PROJECT_TPL_URL" -o "$PACKAGE_DIR/static/project-tpl.zip" + +docker build \ + -f build/Dockerfile \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg http_proxy="$HTTP_PROXY" \ + --build-arg https_proxy="$HTTPS_PROXY" \ + --build-arg no_proxy="$NO_PROXY" \ + --build-arg GOCACHE="$GOCACHE" \ + --build-arg GOMODCACHE="$GOMODCACHE" \ + --build-arg REPO_COMMIT="$REPO_COMMIT" \ + --build-arg BUILD_TARGET=server \ + -t monkeycode-offline/backend:local \ + . +docker build \ + -f build/Dockerfile.ingress \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg http_proxy="$HTTP_PROXY" \ + --build-arg https_proxy="$HTTPS_PROXY" \ + --build-arg no_proxy="$NO_PROXY" \ + -t monkeycode-offline/ingress:local \ + . +docker build \ + -f ../frontend/docker/Dockerfile \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg http_proxy="$HTTP_PROXY" \ + --build-arg https_proxy="$HTTPS_PROXY" \ + --build-arg no_proxy="$NO_PROXY" \ + -t monkeycode-offline/frontend:local \ + ../frontend/docker + +docker pull chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/taskflow:fc0daba +docker pull chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/postgres:17.4-alpine3.21 +docker pull chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/redis:8.0-alpine3.21 +docker pull chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/clickhouse-server:26.3.9 +docker pull chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/rustfs:1.0.0-beta.2 +docker pull chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/codingmatrix-orchestrator:alpha-latest +docker tag chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/taskflow:fc0daba monkeycode-offline/taskflow:local +docker tag chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/postgres:17.4-alpine3.21 monkeycode-offline/postgres:local +docker tag chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/redis:8.0-alpine3.21 monkeycode-offline/redis:local +docker tag chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/clickhouse-server:26.3.9 monkeycode-offline/clickhouse:local +docker tag chaitin-registry.cn-hangzhou.cr.aliyuncs.com/basic/rustfs:1.0.0-beta.2 monkeycode-offline/rustfs:local + +docker save monkeycode-offline/backend:local | gzip > "$PACKAGE_DIR/images/backend.tar.gz" +docker save monkeycode-offline/frontend:local | gzip > "$PACKAGE_DIR/images/frontend.tar.gz" +docker save monkeycode-offline/ingress:local | gzip > "$PACKAGE_DIR/images/ingress.tar.gz" +docker save monkeycode-offline/taskflow:local | gzip > "$PACKAGE_DIR/images/taskflow.tar.gz" +docker save monkeycode-offline/postgres:local | gzip > "$PACKAGE_DIR/images/postgres.tar.gz" +docker save monkeycode-offline/redis:local | gzip > "$PACKAGE_DIR/images/redis.tar.gz" +docker save monkeycode-offline/clickhouse:local | gzip > "$PACKAGE_DIR/images/clickhouse.tar.gz" +docker save monkeycode-offline/rustfs:local | gzip > "$PACKAGE_DIR/images/rustfs.tar.gz" + +build_host_bundle() { + arch="$1" + tmp="$OUT_DIR/host-$arch" + rm -rf "$tmp" + mkdir -p "$tmp/images" + cp Installation/runner/docker-compose.yml "$tmp/docker-compose.yml" + docker save chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode/codingmatrix-orchestrator:alpha-latest | gzip > "$tmp/images/orchestrator.tar.gz" + tar -C "$tmp" -czf "$PACKAGE_DIR/static/installer/$arch/host.tgz" . +} + +build_host_bundle x86_64 +build_host_bundle aarch64 + +scripts/check-offline-package.sh "$PACKAGE_DIR" +tar -C "$OUT_DIR" -czf "$OUT_DIR/$PACKAGE_NAME.tgz" "$PACKAGE_NAME" +echo "$OUT_DIR/monkeycode-offline-linux-$ARCH.tgz" diff --git a/backend/scripts/check-offline-package.sh b/backend/scripts/check-offline-package.sh new file mode 100755 index 00000000..c8fc041a --- /dev/null +++ b/backend/scripts/check-offline-package.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +ROOT="${1:?package dir required}" + +required=" +install.sh +installer +docker.tgz +docker-compose.yml +.env.example +images/backend.tar.gz +images/frontend.tar.gz +images/taskflow.tar.gz +images/ingress.tar.gz +images/postgres.tar.gz +images/redis.tar.gz +images/clickhouse.tar.gz +images/rustfs.tar.gz +static/project-tpl.zip +static/installer/x86_64/installer +static/installer/x86_64/docker.tgz +static/installer/x86_64/host.tgz +static/installer/aarch64/installer +static/installer/aarch64/docker.tgz +static/installer/aarch64/host.tgz +" + +for file in $required; do + if [ ! -f "$ROOT/$file" ]; then + echo "missing $file" + exit 1 + fi +done diff --git a/backend/templates/install_offline.sh.tmpl b/backend/templates/install_offline.sh.tmpl new file mode 100644 index 00000000..e67fc6ec --- /dev/null +++ b/backend/templates/install_offline.sh.tmpl @@ -0,0 +1,79 @@ +#!/bin/bash + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +set -e +trap 'handle_error $? $LINENO' ERR + +handle_error() { + local exit_code=$1 + local line_number=$2 + echo -e "${RED}Error occurred in script at line $line_number with exit code $exit_code${NC}" + cleanup + exit $exit_code +} + +ARCH=$(uname -m) +case "$ARCH" in + "x86_64"|"amd64") + ARCH="x86_64" + ;; + "aarch64"|"arm64"|"armv8l") + ARCH="aarch64" + ;; + *) + echo -e "${RED}This installer only supports amd64 (x86_64) and arm64 (aarch64) architectures${NC}" + echo -e "${RED}Current architecture: $ARCH${NC}" + exit 1 + ;; +esac + +if [[ $EUID -ne 0 ]]; then + echo -e "${RED}This script must be run as root${NC}" + exit 1 +fi + +TOKEN="{{.token}}" +GRPC_URL="{{.grpc_url}}" +BASE_URL="{{.base_url}}" +INSTALLER_URL="{{.installer_url}}" +DOCKER_BUNDLE_PATH="{{.docker_bundle_path}}" +HOST_BUNDLE_PATH="{{.host_bundle_path}}" +INSTALLER_FILE="/tmp/monkeycode-installer" + +cleanup() { + echo -e "${YELLOW}Cleaning up temporary files...${NC}" + rm -f "$INSTALLER_FILE" +} + +if [[ -z "$BASE_URL" || -z "$INSTALLER_URL" ]]; then + echo -e "${RED}Installer URL is empty${NC}" + exit 1 +fi + +echo -e "${GREEN}Architecture: $ARCH${NC}" +echo -e "${GREEN}Starting MonkeyCode Installer Bootstrap${NC}" +echo -e "${YELLOW}Downloading installer...${NC}" + +INSTALLER_URL=${INSTALLER_URL//\{\{.arch\}\}/$ARCH} +DOCKER_BUNDLE_PATH=${DOCKER_BUNDLE_PATH//\{\{.arch\}\}/$ARCH} +HOST_BUNDLE_PATH=${HOST_BUNDLE_PATH//\{\{.arch\}\}/$ARCH} +if ! curl -4fLk --progress-bar -o "$INSTALLER_FILE" "$INSTALLER_URL"; then + echo -e "${RED}Failed to download installer${NC}" + exit 1 +fi + +chmod +x "$INSTALLER_FILE" +MCAI_BASE_URL="$BASE_URL" \ +MCAI_DOCKER_BUNDLE_PATH="$DOCKER_BUNDLE_PATH" \ +MCAI_HOST_BUNDLE_PATH="$HOST_BUNDLE_PATH" \ +MCAI_HOST_TOKEN="$TOKEN" \ +MCAI_TASKFLOW_GRPC_URL="$GRPC_URL" \ +"$INSTALLER_FILE" host + +echo -e "${GREEN}Installer exited successfully${NC}" +cleanup +exit 0 diff --git a/backend/templates/temp.go b/backend/templates/temp.go index 421d365c..a346fc0d 100644 --- a/backend/templates/temp.go +++ b/backend/templates/temp.go @@ -7,6 +7,9 @@ import ( //go:embed install.sh.tmpl var InstallTmpl []byte +//go:embed install_offline.sh.tmpl +var InstallOfflineTmpl []byte + //go:embed codex.tmpl var Codex []byte