From 8277cc91381867ae5d16c89e059f270eb30d2c1a Mon Sep 17 00:00:00 2001 From: wj00037 <1292876134@qq.com> Date: Fri, 8 Nov 2024 09:51:04 +0800 Subject: [PATCH] hwcloud and cdn (#20) * hwcloud and cdn * fix --- config.yml.example | 7 ++-- config/config.go | 7 ++-- go.mod | 14 ++----- go.sum | 24 +----------- main.go | 17 +++------ server/server.go | 94 ++++++++++++++++++++++++++-------------------- 6 files changed, 71 insertions(+), 92 deletions(-) diff --git a/config.yml.example b/config.yml.example index 1ef1e92..e7b0068 100644 --- a/config.yml.example +++ b/config.yml.example @@ -1,7 +1,8 @@ LFS_BUCKET: *********** -AWS_REGION: *********** -AWS_ACCESS_KEY_ID: *********** -AWS_SECRET_ACCESS_KEY: *********** +CDN_DOMAIN: *********** +OBS_REGION: *********** +OBS_ACCESS_KEY_ID: *********** +OBS_SECRET_ACCESS_KEY: *********** CLIENT_ID: *********** CLIENT_SECRET: *********** PATH_PREFIX: *********** \ No newline at end of file diff --git a/config/config.go b/config/config.go index 0285cff..64ad6e6 100644 --- a/config/config.go +++ b/config/config.go @@ -11,9 +11,10 @@ type Config struct { LfsBucket string `json:"LFS_BUCKET"` ClientId string `json:"CLIENT_ID"` ClientSecret string `json:"CLIENT_SECRET"` - AwsRegion string `json:"AWS_REGION"` - AwsAccessKeyId string `json:"AWS_ACCESS_KEY_ID"` - AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"` + CdnDomain string `json:"CDN_DOMAIN"` + ObsRegion string `json:"OBS_REGION"` + ObsAccessKeyId string `json:"OBS_ACCESS_KEY_ID"` + ObsSecretAccessKey string `json:"OBS_SECRET_ACCESS_KEY"` } // LoadConfig loads the configuration file from the specified path and deletes the file if needed diff --git a/go.mod b/go.mod index a157859..025d721 100644 --- a/go.mod +++ b/go.mod @@ -6,23 +6,15 @@ toolchain go1.21.4 require ( github.com/go-chi/chi v4.1.2+incompatible - github.com/minio/minio-go/v7 v7.0.77 sigs.k8s.io/yaml v1.4.0 ) +require github.com/stretchr/testify v1.9.0 // indirect + require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.10 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/rs/xid v1.6.0 // indirect + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible github.com/sirupsen/logrus v1.9.3 - golang.org/x/crypto v0.28.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ce38668..4c1f29a 100644 --- a/go.sum +++ b/go.sum @@ -1,43 +1,23 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= -github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw= -github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible h1:/2MdLc7zHJqzV7J2uVGaoGymVobB/OHC8wmEyWRaK68= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= diff --git a/main.go b/main.go index ded11cd..95eeb3f 100644 --- a/main.go +++ b/main.go @@ -94,22 +94,15 @@ func main() { return } - bucket := cfg.LfsBucket - if bucket == "" { - bucket = os.Getenv("LFS_BUCKET") - if bucket == "" { - logrus.Errorf("LFS_BUCKET must be set") - } - } - s, err := server.New(server.Options{ Prefix: cfg.Prefix, - Bucket: bucket, - Endpoint: cfg.AwsRegion, - AccessKeyID: cfg.AwsAccessKeyId, + Bucket: cfg.LfsBucket, + Endpoint: cfg.ObsRegion, + CdnDomain: cfg.CdnDomain, + AccessKeyID: cfg.ObsAccessKeyId, S3Accelerate: true, IsAuthorized: auth.GiteeAuth(), - SecretAccessKey: cfg.AwsSecretAccessKey, + SecretAccessKey: cfg.ObsSecretAccessKey, }) srv := &http.Server{ Addr: "0.0.0.0:5000", diff --git a/server/server.go b/server/server.go index 241e493..eeb0168 100644 --- a/server/server.go +++ b/server/server.go @@ -6,19 +6,20 @@ import ( "fmt" "math" "net/http" + "net/url" "os" "regexp" "strings" "time" "github.com/go-chi/chi" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" "github.com/metalogical/BigFiles/auth" "github.com/metalogical/BigFiles/batch" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" ) -var S3PutLimit int = 5*int(math.Pow10(9)) - 1 // 5GB - 1 +var ObsPutLimit int = 5*int(math.Pow10(9)) - 1 // 5GB - 1 var oidRegexp = regexp.MustCompile("^[a-f0-9]{64}$") type Options struct { @@ -26,6 +27,7 @@ type Options struct { Endpoint string NoSSL bool Bucket string + CdnDomain string S3Accelerate bool // minio auth (required) @@ -42,22 +44,22 @@ type Options struct { func (o Options) imputeFromEnv() (Options, error) { if o.Endpoint == "" { - region := os.Getenv("AWS_REGION") + region := os.Getenv("OBS_REGION") if region == "" { return o, errors.New("endpoint required") } o.Endpoint = region } if o.AccessKeyID == "" { - o.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") + o.AccessKeyID = os.Getenv("OBS_ACCESS_KEY_ID") if o.AccessKeyID == "" { - return o, fmt.Errorf("AWS access key ID required for %s", o.Endpoint) + return o, fmt.Errorf("OBS access key ID required for %s", o.Endpoint) } - o.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + o.SecretAccessKey = os.Getenv("OBS_SECRET_ACCESS_KEY") if o.SecretAccessKey == "" { - return o, fmt.Errorf("AWS secret access key required for %s", o.Endpoint) + return o, fmt.Errorf("OBS secret access key required for %s", o.Endpoint) } - o.SessionToken = os.Getenv("AWS_SESSION_TOKEN") + o.SessionToken = os.Getenv("OBS_SESSION_TOKEN") } if o.Bucket == "" { return o, fmt.Errorf("bucket required") @@ -75,23 +77,17 @@ func New(o Options) (http.Handler, error) { return nil, err } - // Initialize minio client object. - client, err := minio.New(o.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(o.AccessKeyID, o.SecretAccessKey, o.SessionToken), - Secure: !o.NoSSL, - }) + client, err := obs.New(o.AccessKeyID, o.SecretAccessKey, o.Endpoint, obs.WithSignature(obs.SignatureObs)) if err != nil { return nil, err } - if o.S3Accelerate { - client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com") - } s := &server{ + ttl: o.TTL, client: client, bucket: o.Bucket, prefix: o.Prefix, - ttl: o.TTL, + cdnDomain: o.CdnDomain, isAuthorized: o.IsAuthorized, } @@ -104,10 +100,11 @@ func New(o Options) (http.Handler, error) { } type server struct { - client *minio.Client - bucket string - ttl time.Duration - prefix string + ttl time.Duration + client *obs.ObsClient + bucket string + prefix string + cdnDomain string isAuthorized func(auth.UserInRepo) error } @@ -180,44 +177,52 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { switch req.Operation { case "download": - if info, err := s.client.StatObject(r.Context(), s.bucket, s.key(in.OID), minio.StatObjectOptions{}); err != nil { + getObjectMetadataInput := &obs.GetObjectMetadataInput{ + Bucket: s.bucket, + Key: s.key(in.OID), + } + if metadata, err := s.client.GetObjectMetadata(getObjectMetadataInput); err != nil { out.Error = &batch.ObjectError{ Code: 404, Message: err.Error(), } continue - } else if in.Size != int(info.Size) { + } else if in.Size != int(metadata.ContentLength) { out.Error = &batch.ObjectError{ Code: 422, Message: "found object with wrong size", } } - - href, err := s.client.PresignedGetObject(r.Context(), s.bucket, s.key(in.OID), s.ttl, nil) + getObjectInput := &obs.CreateSignedUrlInput{} + getObjectInput.Method = obs.HttpMethodGet + getObjectInput.Bucket = s.bucket + getObjectInput.Key = s.key(in.OID) + getObjectInput.Expires = int(s.ttl / time.Second) + getObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} + // 生成下载对象的带授权信息的URL + getObjectOutput, err := s.client.CreateSignedUrl(getObjectInput) if err != nil { panic(err) } + v, err := url.Parse(getObjectOutput.SignedUrl) + if err == nil { + v.Host = s.cdnDomain + v.Scheme = "https" + } else { + logrus.Infof("%s cannot be parsed", getObjectOutput.SignedUrl) + panic(err) + } out.Actions = &batch.Actions{ Download: &batch.Action{ - HRef: href.String(), + HRef: v.String(), + Header: getObjectInput.Headers, ExpiresIn: int(s.ttl / time.Second), }, } case "upload": - if info, err := s.client.StatObject(r.Context(), s.bucket, s.key(in.OID), minio.StatObjectOptions{}); err == nil { - if in.Size != int(info.Size) { - out.Error = &batch.ObjectError{ - Code: 422, - Message: "existing object with wrong size", - } - } - // already exists, omit actions - continue - } - - if out.Size > S3PutLimit { + if out.Size > ObsPutLimit { out.Error = &batch.ObjectError{ Code: 422, Message: "cannot upload objects larger than 5GB to S3 via LFS basic transfer adapter", @@ -225,14 +230,21 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { continue } - href, err := s.client.PresignedPutObject(r.Context(), s.bucket, s.key(in.OID), s.ttl) + putObjectInput := &obs.CreateSignedUrlInput{} + putObjectInput.Method = obs.HttpMethodPut + putObjectInput.Bucket = s.bucket + putObjectInput.Key = s.key(in.OID) + putObjectInput.Expires = int(s.ttl / time.Second) + putObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} + putObjectOutput, err := s.client.CreateSignedUrl(putObjectInput) if err != nil { panic(err) } out.Actions = &batch.Actions{ Upload: &batch.Action{ - HRef: href.String(), + HRef: putObjectOutput.SignedUrl, + Header: putObjectInput.Headers, ExpiresIn: int(s.ttl / time.Second), }, }