From d0f5ecb6d235063223cb761f2213fee4b2fa1228 Mon Sep 17 00:00:00 2001 From: HynoR <20227709+HynoR@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:25:36 +0800 Subject: [PATCH 1/2] chore: update dependencies and migrate S3 client to MinIO SDK - Removed AWS SDK dependency and replaced S3 client implementation with MinIO SDK. - Updated various dependencies in go.mod and go.sum, including version upgrades for `klauspost/compress`, `minio/minio-go`, and `rs/xid`. - Adjusted S3 client methods to utilize MinIO's API for bucket listing, object existence checks, uploads, downloads, and deletions. --- agent/go.mod | 16 ++-- agent/go.sum | 28 +++--- agent/utils/cloud_storage/client/s3.go | 125 +++++++++++-------------- 3 files changed, 79 insertions(+), 90 deletions(-) diff --git a/agent/go.mod b/agent/go.mod index e85541c8f9d7..90165c9935ff 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -4,9 +4,9 @@ go 1.24.9 require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible - github.com/aws/aws-sdk-go v1.55.0 github.com/compose-spec/compose-go/v2 v2.9.0 github.com/creack/pty v1.1.24 + github.com/docker/cli v28.5.1+incompatible github.com/docker/compose/v2 v2.40.2 github.com/docker/docker v28.5.1+incompatible github.com/docker/go-connections v0.6.0 @@ -26,10 +26,10 @@ require ( github.com/jinzhu/copier v0.4.0 github.com/jinzhu/gorm v1.9.16 github.com/joho/godotenv v1.5.1 - github.com/klauspost/compress v1.18.1 + github.com/klauspost/compress v1.18.2 github.com/mholt/archiver/v4 v4.0.0-alpha.8 github.com/miekg/dns v1.1.69 - github.com/minio/minio-go/v7 v7.0.74 + github.com/minio/minio-go/v7 v7.0.98 github.com/nicksnyder/go-i18n/v2 v2.4.0 github.com/opencontainers/image-spec v1.1.1 github.com/oschwald/maxminddb-golang v1.13.1 @@ -113,7 +113,6 @@ require ( github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/buildx v0.29.1 // indirect - github.com/docker/cli v28.5.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.1 // indirect @@ -159,9 +158,9 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -172,6 +171,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -202,12 +202,13 @@ require ( github.com/ovh/go-ovh v1.9.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/rs/xid v1.5.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect @@ -218,6 +219,7 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.3.12 // indirect github.com/therootcompany/xz v1.0.1 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/tjfoc/gmsm v1.4.1 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index 3c9be6a1f297..030b37c21a2e 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -128,8 +128,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.55.0 h1:hVALKPjXz33kP1R9nTyJpUK7qF59dO2mleQxUW9mCVE= -github.com/aws/aws-sdk-go v1.55.0/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= @@ -588,9 +586,7 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= @@ -613,13 +609,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -683,12 +681,14 @@ github.com/miekg/dns v1.1.69 h1:Kb7Y/1Jo+SG+a2GtfoFUfDkG//csdRPwRLkCsxDG9Sc= github.com/miekg/dns v1.1.69/go.mod h1:7OyjD9nEba5OkqQ/hB4fy3PIoxafSZJtducccIelz3g= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= -github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -804,6 +804,8 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= @@ -871,8 +873,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -951,6 +953,8 @@ github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+x github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= diff --git a/agent/utils/cloud_storage/client/s3.go b/agent/utils/cloud_storage/client/s3.go index 3a00647146d7..2124a7dd0407 100644 --- a/agent/utils/cloud_storage/client/s3.go +++ b/agent/utils/cloud_storage/client/s3.go @@ -1,20 +1,18 @@ package client import ( + "context" "os" + "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) type s3Client struct { scType string bucket string - Sess session.Session + client *minio.Client } func NewS3Client(vars map[string]interface{}) (*s3Client, error) { @@ -31,69 +29,61 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) { if len(mode) == 0 { mode = "virtual hosted" } - sess, err := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), - Endpoint: aws.String(endpoint), - Region: aws.String(region), - DisableSSL: aws.Bool(true), S3ForcePathStyle: aws.Bool(mode == "path"), + + lookupStyle := minio.BucketLookupDNS + if mode == "path" { + lookupStyle = minio.BucketLookupPath + } + + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: false, + Region: region, + BucketLookup: lookupStyle, }) if err != nil { return nil, err } - return &s3Client{scType: scType, bucket: bucket, Sess: *sess}, nil + return &s3Client{scType: scType, bucket: bucket, client: client}, nil } func (s s3Client) ListBuckets() ([]interface{}, error) { - var result []interface{} - svc := s3.New(&s.Sess) - res, err := svc.ListBuckets(nil) + buckets, err := s.client.ListBuckets(context.Background()) if err != nil { return nil, err } - for _, b := range res.Buckets { + var result []interface{} + for _, b := range buckets { result = append(result, b.Name) } return result, nil } func (s s3Client) Exist(path string) (bool, error) { - svc := s3.New(&s.Sess) - if _, err := svc.HeadObject(&s3.HeadObjectInput{ - Bucket: &s.bucket, - Key: &path, - }); err != nil { - if aerr, ok := err.(awserr.RequestFailure); ok { - if aerr.StatusCode() == 404 { - return false, nil - } - } else { - return false, aerr + _, err := s.client.StatObject(context.Background(), s.bucket, path, minio.StatObjectOptions{}) + if err != nil { + resp := minio.ToErrorResponse(err) + if resp.StatusCode == 404 { + return false, nil } + return false, err } return true, nil } func (s *s3Client) Size(path string) (int64, error) { - svc := s3.New(&s.Sess) - file, err := svc.GetObject(&s3.GetObjectInput{ - Bucket: &s.bucket, - Key: &path, - }) + info, err := s.client.StatObject(context.Background(), s.bucket, path, minio.StatObjectOptions{}) if err != nil { return 0, err } - return *file.ContentLength, nil + return info.Size, nil } func (s s3Client) Delete(path string) (bool, error) { - svc := s3.New(&s.Sess) - if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil { - return false, err - } - if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(path), - }); err != nil { + if err := s.client.RemoveObject(context.Background(), s.bucket, path, minio.RemoveObjectOptions{}); err != nil { return false, err } return true, nil @@ -110,16 +100,19 @@ func (s s3Client) Upload(src, target string) (bool, error) { } defer file.Close() - uploader := s3manager.NewUploader(&s.Sess) - if fileInfo.Size() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = fileInfo.Size() / (s3manager.MaxUploadParts - 1) + opts := minio.PutObjectOptions{ + StorageClass: s.scType, } - if _, err := uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(target), - Body: file, - StorageClass: &s.scType, - }); err != nil { + + const maxParts = 10000 + const defaultPartSize = 64 * 1024 * 1024 // 64 MiB + partSize := uint64(defaultPartSize) + if fileInfo.Size() > int64(maxParts)*int64(defaultPartSize) { + partSize = uint64(fileInfo.Size()) / (maxParts - 1) + } + opts.PartSize = partSize + + if _, err := s.client.PutObject(context.Background(), s.bucket, target, file, fileInfo.Size(), opts); err != nil { return false, err } return true, nil @@ -129,16 +122,7 @@ func (s s3Client) Download(src, target string) (bool, error) { if _, err := os.Stat(target); err == nil { _ = os.Remove(target) } - file, err := os.Create(target) - if err != nil { - return false, err - } - defer file.Close() - downloader := s3manager.NewDownloader(&s.Sess) - if _, err = downloader.Download(file, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(src), - }); err != nil { + if err := s.client.FGetObject(context.Background(), s.bucket, src, target, minio.GetObjectOptions{}); err != nil { os.Remove(target) return false, err } @@ -146,17 +130,16 @@ func (s s3Client) Download(src, target string) (bool, error) { } func (s *s3Client) ListObjects(prefix string) ([]string, error) { - svc := s3.New(&s.Sess) - var result []string - outputs, err := svc.ListObjects(&s3.ListObjectsInput{ - Bucket: &s.bucket, - Prefix: &prefix, - }) - if err != nil { - return result, err + opts := minio.ListObjectsOptions{ + Recursive: true, + Prefix: prefix, } - for _, item := range outputs.Contents { - result = append(result, *item.Key) + var result []string + for object := range s.client.ListObjects(context.Background(), s.bucket, opts) { + if object.Err != nil { + return result, object.Err + } + result = append(result, object.Key) } return result, nil } From 8bb93a4ce6064953c1faaf9ce5b64fa1ef4256f6 Mon Sep 17 00:00:00 2001 From: HynoR <20227709+HynoR@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:38:07 +0800 Subject: [PATCH 2/2] refactor: enhance S3 client with context management and TLS support - Introduced context management with timeouts for S3 client operations to improve reliability. - Added TLS configuration to support secure connections based on the endpoint scheme. - Updated S3 client methods to utilize the new context and transport settings for enhanced performance and security. --- agent/utils/cloud_storage/client/s3.go | 57 +++++++++++++++++++++----- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/agent/utils/cloud_storage/client/s3.go b/agent/utils/cloud_storage/client/s3.go index 2124a7dd0407..2f85bda3cd52 100644 --- a/agent/utils/cloud_storage/client/s3.go +++ b/agent/utils/cloud_storage/client/s3.go @@ -2,19 +2,31 @@ package client import ( "context" + "crypto/tls" + "net/http" "os" "strings" + "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) +const ( + s3DefaultTimeout = 30 * time.Second + s3TransferTimeout = 24 * time.Hour +) + type s3Client struct { scType string bucket string client *minio.Client } +func (s *s3Client) ctx(timeout time.Duration) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), timeout) +} + func NewS3Client(vars map[string]interface{}) (*s3Client, error) { accessKey := loadParamFromVars("accessKey", vars) secretKey := loadParamFromVars("secretKey", vars) @@ -35,14 +47,25 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) { lookupStyle = minio.BucketLookupPath } - endpoint = strings.TrimPrefix(endpoint, "https://") - endpoint = strings.TrimPrefix(endpoint, "http://") + ssl := strings.Split(endpoint, ":")[0] + secure := false + tlsConfig := &tls.Config{} + if ssl == "https" { + secure = true + tlsConfig.InsecureSkipVerify = true + } + var transport http.RoundTripper = &http.Transport{ + TLSClientConfig: tlsConfig, + } + + endpoint = strings.TrimPrefix(endpoint, ssl+"://") client, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKey, secretKey, ""), - Secure: false, + Secure: secure, Region: region, BucketLookup: lookupStyle, + Transport: transport, }) if err != nil { return nil, err @@ -51,7 +74,9 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) { } func (s s3Client) ListBuckets() ([]interface{}, error) { - buckets, err := s.client.ListBuckets(context.Background()) + ctx, cancel := s.ctx(s3DefaultTimeout) + defer cancel() + buckets, err := s.client.ListBuckets(ctx) if err != nil { return nil, err } @@ -63,7 +88,9 @@ func (s s3Client) ListBuckets() ([]interface{}, error) { } func (s s3Client) Exist(path string) (bool, error) { - _, err := s.client.StatObject(context.Background(), s.bucket, path, minio.StatObjectOptions{}) + ctx, cancel := s.ctx(s3DefaultTimeout) + defer cancel() + _, err := s.client.StatObject(ctx, s.bucket, path, minio.StatObjectOptions{}) if err != nil { resp := minio.ToErrorResponse(err) if resp.StatusCode == 404 { @@ -75,7 +102,9 @@ func (s s3Client) Exist(path string) (bool, error) { } func (s *s3Client) Size(path string) (int64, error) { - info, err := s.client.StatObject(context.Background(), s.bucket, path, minio.StatObjectOptions{}) + ctx, cancel := s.ctx(s3DefaultTimeout) + defer cancel() + info, err := s.client.StatObject(ctx, s.bucket, path, minio.StatObjectOptions{}) if err != nil { return 0, err } @@ -83,7 +112,9 @@ func (s *s3Client) Size(path string) (int64, error) { } func (s s3Client) Delete(path string) (bool, error) { - if err := s.client.RemoveObject(context.Background(), s.bucket, path, minio.RemoveObjectOptions{}); err != nil { + ctx, cancel := s.ctx(s3DefaultTimeout) + defer cancel() + if err := s.client.RemoveObject(ctx, s.bucket, path, minio.RemoveObjectOptions{}); err != nil { return false, err } return true, nil @@ -112,7 +143,9 @@ func (s s3Client) Upload(src, target string) (bool, error) { } opts.PartSize = partSize - if _, err := s.client.PutObject(context.Background(), s.bucket, target, file, fileInfo.Size(), opts); err != nil { + ctx, cancel := s.ctx(s3TransferTimeout) + defer cancel() + if _, err := s.client.PutObject(ctx, s.bucket, target, file, fileInfo.Size(), opts); err != nil { return false, err } return true, nil @@ -122,7 +155,9 @@ func (s s3Client) Download(src, target string) (bool, error) { if _, err := os.Stat(target); err == nil { _ = os.Remove(target) } - if err := s.client.FGetObject(context.Background(), s.bucket, src, target, minio.GetObjectOptions{}); err != nil { + ctx, cancel := s.ctx(s3TransferTimeout) + defer cancel() + if err := s.client.FGetObject(ctx, s.bucket, src, target, minio.GetObjectOptions{}); err != nil { os.Remove(target) return false, err } @@ -135,7 +170,9 @@ func (s *s3Client) ListObjects(prefix string) ([]string, error) { Prefix: prefix, } var result []string - for object := range s.client.ListObjects(context.Background(), s.bucket, opts) { + ctx, cancel := s.ctx(s3DefaultTimeout) + defer cancel() + for object := range s.client.ListObjects(ctx, s.bucket, opts) { if object.Err != nil { return result, object.Err }