hwcloud and cdn (#20)

* hwcloud and cdn

* fix

config.yml.example

@@ -1,7 +1,8 @@
 LFS_BUCKET: ***********
-AWS_REGION: ***********
-AWS_ACCESS_KEY_ID: ***********
-AWS_SECRET_ACCESS_KEY: ***********
+CDN_DOMAIN: ***********
+OBS_REGION: ***********
+OBS_ACCESS_KEY_ID: ***********
+OBS_SECRET_ACCESS_KEY: ***********
 CLIENT_ID: ***********
 CLIENT_SECRET: ***********
 PATH_PREFIX: ***********

config/config.go

@@ -11,9 +11,10 @@ type Config struct {
 	LfsBucket          string `json:"LFS_BUCKET"`
 	ClientId           string `json:"CLIENT_ID"`
 	ClientSecret       string `json:"CLIENT_SECRET"`
-	AwsRegion          string `json:"AWS_REGION"`
-	AwsAccessKeyId     string `json:"AWS_ACCESS_KEY_ID"`
-	AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"`
+	CdnDomain          string `json:"CDN_DOMAIN"`
+	ObsRegion          string `json:"OBS_REGION"`
+	ObsAccessKeyId     string `json:"OBS_ACCESS_KEY_ID"`
+	ObsSecretAccessKey string `json:"OBS_SECRET_ACCESS_KEY"`
 }
 
 // LoadConfig loads the configuration file from the specified path and deletes the file if needed

go.mod

@@ -6,23 +6,15 @@ toolchain go1.21.4
 
 require (
 	github.com/go-chi/chi v4.1.2+incompatible
-	github.com/minio/minio-go/v7 v7.0.77
 	sigs.k8s.io/yaml v1.4.0
 )
 
+require github.com/stretchr/testify v1.9.0 // indirect
+
 require (
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/goccy/go-json v0.10.3 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/klauspost/compress v1.17.10 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
-	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/rs/xid v1.6.0 // indirect
+	github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible
 	github.com/sirupsen/logrus v1.9.3
-	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,43 +1,23 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
-github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
-github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
-github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
-github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
-github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
-github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw=
-github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=
+github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible h1:/2MdLc7zHJqzV7J2uVGaoGymVobB/OHC8wmEyWRaK68=
+github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
-github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
 golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=

main.go

@@ -94,22 +94,15 @@ func main() {
 		return
 	}
 
-	bucket := cfg.LfsBucket
-	if bucket == "" {
-		bucket = os.Getenv("LFS_BUCKET")
-		if bucket == "" {
-			logrus.Errorf("LFS_BUCKET must be set")
-		}
-	}
-
 	s, err := server.New(server.Options{
 		Prefix:          cfg.Prefix,
-		Bucket:          bucket,
-		Endpoint:        cfg.AwsRegion,
-		AccessKeyID:     cfg.AwsAccessKeyId,
+		Bucket:          cfg.LfsBucket,
+		Endpoint:        cfg.ObsRegion,
+		CdnDomain:       cfg.CdnDomain,
+		AccessKeyID:     cfg.ObsAccessKeyId,
 		S3Accelerate:    true,
 		IsAuthorized:    auth.GiteeAuth(),
-		SecretAccessKey: cfg.AwsSecretAccessKey,
+		SecretAccessKey: cfg.ObsSecretAccessKey,
 	})
 	srv := &http.Server{
 		Addr:         "0.0.0.0:5000",

server/server.go

@@ -6,26 +6,28 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"net/url"
 	"os"
 	"regexp"
 	"strings"
 	"time"
 
 	"github.com/go-chi/chi"
+	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
 	"github.com/metalogical/BigFiles/auth"
 	"github.com/metalogical/BigFiles/batch"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/sirupsen/logrus"
 )
 
-var S3PutLimit int = 5*int(math.Pow10(9)) - 1 // 5GB - 1
+var ObsPutLimit int = 5*int(math.Pow10(9)) - 1 // 5GB - 1
 var oidRegexp = regexp.MustCompile("^[a-f0-9]{64}$")
 
 type Options struct {
 	// required
 	Endpoint     string
 	NoSSL        bool
 	Bucket       string
+	CdnDomain    string
 	S3Accelerate bool
 
 	// minio auth (required)
@@ -42,22 +44,22 @@ type Options struct {
 
 func (o Options) imputeFromEnv() (Options, error) {
 	if o.Endpoint == "" {
-		region := os.Getenv("AWS_REGION")
+		region := os.Getenv("OBS_REGION")
 		if region == "" {
 			return o, errors.New("endpoint required")
 		}
 		o.Endpoint = region
 	}
 	if o.AccessKeyID == "" {
-		o.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
+		o.AccessKeyID = os.Getenv("OBS_ACCESS_KEY_ID")
 		if o.AccessKeyID == "" {
-			return o, fmt.Errorf("AWS access key ID required for %s", o.Endpoint)
+			return o, fmt.Errorf("OBS access key ID required for %s", o.Endpoint)
 		}
-		o.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
+		o.SecretAccessKey = os.Getenv("OBS_SECRET_ACCESS_KEY")
 		if o.SecretAccessKey == "" {
-			return o, fmt.Errorf("AWS secret access key required for %s", o.Endpoint)
+			return o, fmt.Errorf("OBS secret access key required for %s", o.Endpoint)
 		}
-		o.SessionToken = os.Getenv("AWS_SESSION_TOKEN")
+		o.SessionToken = os.Getenv("OBS_SESSION_TOKEN")
 	}
 	if o.Bucket == "" {
 		return o, fmt.Errorf("bucket required")
@@ -75,23 +77,17 @@ func New(o Options) (http.Handler, error) {
 		return nil, err
 	}
 
-	// Initialize minio client object.
-	client, err := minio.New(o.Endpoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(o.AccessKeyID, o.SecretAccessKey, o.SessionToken),
-		Secure: !o.NoSSL,
-	})
+	client, err := obs.New(o.AccessKeyID, o.SecretAccessKey, o.Endpoint, obs.WithSignature(obs.SignatureObs))
 	if err != nil {
 		return nil, err
 	}
-	if o.S3Accelerate {
-		client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com")
-	}
 
 	s := &server{
+		ttl:          o.TTL,
 		client:       client,
 		bucket:       o.Bucket,
 		prefix:       o.Prefix,
-		ttl:          o.TTL,
+		cdnDomain:    o.CdnDomain,
 		isAuthorized: o.IsAuthorized,
 	}
 
@@ -104,10 +100,11 @@ func New(o Options) (http.Handler, error) {
 }
 
 type server struct {
-	client *minio.Client
-	bucket string
-	ttl    time.Duration
-	prefix string
+	ttl       time.Duration
+	client    *obs.ObsClient
+	bucket    string
+	prefix    string
+	cdnDomain string
 
 	isAuthorized func(auth.UserInRepo) error
 }
@@ -180,59 +177,74 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) {
 
 		switch req.Operation {
 		case "download":
-			if info, err := s.client.StatObject(r.Context(), s.bucket, s.key(in.OID), minio.StatObjectOptions{}); err != nil {
+			getObjectMetadataInput := &obs.GetObjectMetadataInput{
+				Bucket: s.bucket,
+				Key:    s.key(in.OID),
+			}
+			if metadata, err := s.client.GetObjectMetadata(getObjectMetadataInput); err != nil {
 				out.Error = &batch.ObjectError{
 					Code:    404,
 					Message: err.Error(),
 				}
 				continue
-			} else if in.Size != int(info.Size) {
+			} else if in.Size != int(metadata.ContentLength) {
 				out.Error = &batch.ObjectError{
 					Code:    422,
 					Message: "found object with wrong size",
 				}
 			}
-
-			href, err := s.client.PresignedGetObject(r.Context(), s.bucket, s.key(in.OID), s.ttl, nil)
+			getObjectInput := &obs.CreateSignedUrlInput{}
+			getObjectInput.Method = obs.HttpMethodGet
+			getObjectInput.Bucket = s.bucket
+			getObjectInput.Key = s.key(in.OID)
+			getObjectInput.Expires = int(s.ttl / time.Second)
+			getObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"}
+			// 生成下载对象的带授权信息的URL
+			getObjectOutput, err := s.client.CreateSignedUrl(getObjectInput)
 			if err != nil {
 				panic(err)
 			}
+			v, err := url.Parse(getObjectOutput.SignedUrl)
+			if err == nil {
+				v.Host = s.cdnDomain
+				v.Scheme = "https"
+			} else {
+				logrus.Infof("%s cannot be parsed", getObjectOutput.SignedUrl)
+				panic(err)
+			}
 
 			out.Actions = &batch.Actions{
 				Download: &batch.Action{
-					HRef:      href.String(),
+					HRef:      v.String(),
+					Header:    getObjectInput.Headers,
 					ExpiresIn: int(s.ttl / time.Second),
 				},
 			}
 
 		case "upload":
-			if info, err := s.client.StatObject(r.Context(), s.bucket, s.key(in.OID), minio.StatObjectOptions{}); err == nil {
-				if in.Size != int(info.Size) {
-					out.Error = &batch.ObjectError{
-						Code:    422,
-						Message: "existing object with wrong size",
-					}
-				}
-				// already exists, omit actions
-				continue
-			}
-
-			if out.Size > S3PutLimit {
+			if out.Size > ObsPutLimit {
 				out.Error = &batch.ObjectError{
 					Code:    422,
 					Message: "cannot upload objects larger than 5GB to S3 via LFS basic transfer adapter",
 				}
 				continue
 			}
 
-			href, err := s.client.PresignedPutObject(r.Context(), s.bucket, s.key(in.OID), s.ttl)
+			putObjectInput := &obs.CreateSignedUrlInput{}
+			putObjectInput.Method = obs.HttpMethodPut
+			putObjectInput.Bucket = s.bucket
+			putObjectInput.Key = s.key(in.OID)
+			putObjectInput.Expires = int(s.ttl / time.Second)
+			putObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"}
+			putObjectOutput, err := s.client.CreateSignedUrl(putObjectInput)
 			if err != nil {
 				panic(err)
 			}
 
 			out.Actions = &batch.Actions{
 				Upload: &batch.Action{
-					HRef:      href.String(),
+					HRef:      putObjectOutput.SignedUrl,
+					Header:    putObjectInput.Headers,
 					ExpiresIn: int(s.ttl / time.Second),
 				},
 			}