From da8f4fa362ba07ede62b1545aebca9d0272f8c52 Mon Sep 17 00:00:00 2001 From: Zherphy <1123678689@qq.com> Date: Fri, 8 Nov 2024 17:11:10 +0800 Subject: [PATCH 1/3] Fix complexity problem (#21) * fix: fix complexity problem fix complexity problem * fix: fix complexity problem fix complexity problem generateDownloadUrl * fix: fix complexity problem fix complexity problem generateDownloadUrl * add: fix-complexity-problem add downloadObject uploadObject * add: fix-complexity-problem fix ci error --------- Co-authored-by: Zherphy --- batch/types.go | 10 +-- server/server.go | 158 +++++++++++++++++++++++++---------------------- 2 files changed, 91 insertions(+), 77 deletions(-) diff --git a/batch/types.go b/batch/types.go index 9d97f6e..fc58c13 100644 --- a/batch/types.go +++ b/batch/types.go @@ -8,10 +8,12 @@ type Request struct { Ref struct { Name string `json:"name"` } `json:"ref"` - Objects []struct { - OID string `json:"oid"` - Size int `json:"size"` - } `json:"objects"` + Objects []RequestObject `json:"objects"` +} + +type RequestObject struct { + OID string `json:"oid"` + Size int `json:"size"` } type SuccessResponse struct { diff --git a/server/server.go b/server/server.go index eeb0168..068813f 100644 --- a/server/server.go +++ b/server/server.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/sirupsen/logrus" "math" "net/http" "net/url" @@ -16,7 +17,6 @@ import ( "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" "github.com/metalogical/BigFiles/auth" "github.com/metalogical/BigFiles/batch" - "github.com/sirupsen/logrus" ) var ObsPutLimit int = 5*int(math.Pow10(9)) - 1 // 5GB - 1 @@ -157,10 +157,9 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { return } } - var resp batch.Response - - for _, in := range req.Objects { + for i := 0; i < len(req.Objects); i++ { + in := req.Objects[i] resp.Objects = append(resp.Objects, batch.Object{ OID: in.OID, Size: in.Size, @@ -177,81 +176,94 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { switch req.Operation { case "download": - getObjectMetadataInput := &obs.GetObjectMetadataInput{ - Bucket: s.bucket, - Key: s.key(in.OID), - } - if metadata, err := s.client.GetObjectMetadata(getObjectMetadataInput); err != nil { - out.Error = &batch.ObjectError{ - Code: 404, - Message: err.Error(), - } - continue - } else if in.Size != int(metadata.ContentLength) { - out.Error = &batch.ObjectError{ - Code: 422, - Message: "found object with wrong size", - } - } - getObjectInput := &obs.CreateSignedUrlInput{} - getObjectInput.Method = obs.HttpMethodGet - getObjectInput.Bucket = s.bucket - getObjectInput.Key = s.key(in.OID) - getObjectInput.Expires = int(s.ttl / time.Second) - getObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} - // 生成下载对象的带授权信息的URL - getObjectOutput, err := s.client.CreateSignedUrl(getObjectInput) - if err != nil { - panic(err) - } - v, err := url.Parse(getObjectOutput.SignedUrl) - if err == nil { - v.Host = s.cdnDomain - v.Scheme = "https" - } else { - logrus.Infof("%s cannot be parsed", getObjectOutput.SignedUrl) - panic(err) - } - - out.Actions = &batch.Actions{ - Download: &batch.Action{ - HRef: v.String(), - Header: getObjectInput.Headers, - ExpiresIn: int(s.ttl / time.Second), - }, - } - + s.downloadObject(&in, out) case "upload": - if out.Size > ObsPutLimit { - out.Error = &batch.ObjectError{ - Code: 422, - Message: "cannot upload objects larger than 5GB to S3 via LFS basic transfer adapter", - } - continue - } + s.uploadObject(&in, out) + } + } + must(json.NewEncoder(w).Encode(resp)) +} - putObjectInput := &obs.CreateSignedUrlInput{} - putObjectInput.Method = obs.HttpMethodPut - putObjectInput.Bucket = s.bucket - putObjectInput.Key = s.key(in.OID) - putObjectInput.Expires = int(s.ttl / time.Second) - putObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} - putObjectOutput, err := s.client.CreateSignedUrl(putObjectInput) - if err != nil { - panic(err) - } +func (s *server) downloadObject(in *batch.RequestObject, out *batch.Object) { + getObjectMetadataInput := &obs.GetObjectMetadataInput{ + Bucket: s.bucket, + Key: s.key(in.OID), + } + if metadata, err := s.client.GetObjectMetadata(getObjectMetadataInput); err != nil { + out.Error = &batch.ObjectError{ + Code: 404, + Message: err.Error(), + } + return + } else if in.Size != int(metadata.ContentLength) { + out.Error = &batch.ObjectError{ + Code: 422, + Message: "found object with wrong size", + } + } + getObjectInput := &obs.CreateSignedUrlInput{} + getObjectInput.Method = obs.HttpMethodGet + getObjectInput.Bucket = s.bucket + getObjectInput.Key = s.key(in.OID) + getObjectInput.Expires = int(s.ttl / time.Second) + getObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} + // 生成下载对象的带授权信息的URL + v := s.generateDownloadUrl(getObjectInput) + + out.Actions = &batch.Actions{ + Download: &batch.Action{ + HRef: v.String(), + Header: getObjectInput.Headers, + ExpiresIn: int(s.ttl / time.Second), + }, + } +} - out.Actions = &batch.Actions{ - Upload: &batch.Action{ - HRef: putObjectOutput.SignedUrl, - Header: putObjectInput.Headers, - ExpiresIn: int(s.ttl / time.Second), - }, - } +func (s *server) uploadObject(in *batch.RequestObject, out *batch.Object) { + if out.Size > ObsPutLimit { + out.Error = &batch.ObjectError{ + Code: 422, + Message: "cannot upload objects larger than 5GB to S3 via LFS basic transfer adapter", } + return } - must(json.NewEncoder(w).Encode(resp)) + putObjectInput := &obs.CreateSignedUrlInput{} + putObjectInput.Method = obs.HttpMethodPut + putObjectInput.Bucket = s.bucket + putObjectInput.Key = s.key(in.OID) + putObjectInput.Expires = int(s.ttl / time.Second) + putObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} + putObjectOutput, err := s.client.CreateSignedUrl(putObjectInput) + if err != nil { + panic(err) + } + + out.Actions = &batch.Actions{ + Upload: &batch.Action{ + HRef: putObjectOutput.SignedUrl, + Header: putObjectInput.Headers, + ExpiresIn: int(s.ttl / time.Second), + }, + } +} + +// 生成下载对象的带授权信息的URL +func (s *server) generateDownloadUrl(getObjectInput *obs.CreateSignedUrlInput) *url.URL { + // 生成下载对象的带授权信息的URL + getObjectOutput, err := s.client.CreateSignedUrl(getObjectInput) + if err != nil { + panic(err) + } + v, err := url.Parse(getObjectOutput.SignedUrl) + if err == nil { + v.Host = s.cdnDomain + v.Scheme = "https" + } else { + logrus.Infof("%s cannot be parsed", getObjectOutput.SignedUrl) + panic(err) + } + return v } func (s *server) healthCheck(w http.ResponseWriter, r *http.Request) { From a9633681899473fac9c79a7adc6401e7a2e07f4c Mon Sep 17 00:00:00 2001 From: wj00037 <1292876134@qq.com> Date: Fri, 8 Nov 2024 17:11:22 +0800 Subject: [PATCH 2/3] add auth tests (#22) * add auth tests * fix --- auth/gitee_test.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 7 ++++- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 auth/gitee_test.go diff --git a/auth/gitee_test.go b/auth/gitee_test.go new file mode 100644 index 0000000..fb892c9 --- /dev/null +++ b/auth/gitee_test.go @@ -0,0 +1,66 @@ +package auth + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// SuiteUserInRepo used for testing +type SuiteUserInRepo struct { + suite.Suite + Repo string + Owner string +} + +// SetupSuite used for testing +func (s *SuiteUserInRepo) SetupSuite() { + s.Repo = "software-package-server" + s.Owner = "src-openeuler" +} + +// TearDownSuite used for testing +func (s *SuiteUserInRepo) TearDownSuite() { + +} + +func (s *SuiteUserInRepo) TestGetToken() { + // getToken fail + token, err := getToken("user", "wrong_pwd") + assert.Equal(s.T(), "", token) + assert.NotNil(s.T(), err.Error()) +} + +func (s *SuiteUserInRepo) TestCheckRepoOwner() { + // CheckRepoOwner success + userInRepo := UserInRepo{ + Repo: s.Repo, + Owner: s.Owner, + } + err := CheckRepoOwner(userInRepo) + assert.Nil(s.T(), err) + + // check no_exist repo + userInRepo = UserInRepo{ + Repo: "repo", + Owner: "owner", + } + err = CheckRepoOwner(userInRepo) + assert.NotNil(s.T(), err) +} + +func (s *SuiteUserInRepo) TestVerifyUser() { + userInRepo := UserInRepo{ + Repo: s.Repo, + Owner: s.Owner, + Operation: "download", + } + + err := verifyUser(userInRepo) + assert.NotNil(s.T(), err) +} + +func TestGitee(t *testing.T) { + suite.Run(t, new(SuiteUserInRepo)) +} diff --git a/go.mod b/go.mod index 025d721..dd888e1 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,12 @@ require ( sigs.k8s.io/yaml v1.4.0 ) -require github.com/stretchr/testify v1.9.0 // indirect +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 + gopkg.in/yaml.v3 v3.0.1 // indirect +) require ( github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.6+incompatible From 10beb8f22a837982d0c4882774f2091bbedf2a74 Mon Sep 17 00:00:00 2001 From: wj00037 <1292876134@qq.com> Date: Fri, 8 Nov 2024 17:47:23 +0800 Subject: [PATCH 3/3] add github tests (#23) --- .github/workflows/lint.yml | 2 +- auth/gitee_test.go | 17 +++++++------- auth/github_test.go | 47 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 auth/github_test.go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3f61c18..57ecdcd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,6 +41,6 @@ jobs: with: args: -exclude-dir BigFiles -exclude-dir utils ./... - name: Unit tests - run: go test -race -count=1 -v ./... + run: go test -race -count=1 -cover -v ./... - name: cleanup run: rm -f ~/.netrc \ No newline at end of file diff --git a/auth/gitee_test.go b/auth/gitee_test.go index fb892c9..e5f085a 100644 --- a/auth/gitee_test.go +++ b/auth/gitee_test.go @@ -7,32 +7,31 @@ import ( "github.com/stretchr/testify/suite" ) -// SuiteUserInRepo used for testing -type SuiteUserInRepo struct { +// SuiteGitee used for testing +type SuiteGitee struct { suite.Suite Repo string Owner string } // SetupSuite used for testing -func (s *SuiteUserInRepo) SetupSuite() { +func (s *SuiteGitee) SetupSuite() { s.Repo = "software-package-server" s.Owner = "src-openeuler" } // TearDownSuite used for testing -func (s *SuiteUserInRepo) TearDownSuite() { - +func (s *SuiteGitee) TearDownSuite() { } -func (s *SuiteUserInRepo) TestGetToken() { +func (s *SuiteGitee) TestGetToken() { // getToken fail token, err := getToken("user", "wrong_pwd") assert.Equal(s.T(), "", token) assert.NotNil(s.T(), err.Error()) } -func (s *SuiteUserInRepo) TestCheckRepoOwner() { +func (s *SuiteGitee) TestCheckRepoOwner() { // CheckRepoOwner success userInRepo := UserInRepo{ Repo: s.Repo, @@ -50,7 +49,7 @@ func (s *SuiteUserInRepo) TestCheckRepoOwner() { assert.NotNil(s.T(), err) } -func (s *SuiteUserInRepo) TestVerifyUser() { +func (s *SuiteGitee) TestVerifyUser() { userInRepo := UserInRepo{ Repo: s.Repo, Owner: s.Owner, @@ -62,5 +61,5 @@ func (s *SuiteUserInRepo) TestVerifyUser() { } func TestGitee(t *testing.T) { - suite.Run(t, new(SuiteUserInRepo)) + suite.Run(t, new(SuiteGitee)) } diff --git a/auth/github_test.go b/auth/github_test.go new file mode 100644 index 0000000..1c65c84 --- /dev/null +++ b/auth/github_test.go @@ -0,0 +1,47 @@ +package auth + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +// SuiteGithub used for testing +type SuiteGithub struct { + suite.Suite + Username string + Password string +} + +// SetupSuite used for testing +func (s *SuiteGithub) SetupSuite() { + s.Username = "username" + s.Password = "password" +} + +// TearDownSuite used for testing +func (s *SuiteGithub) TearDownSuite() { +} + +func (s *SuiteGithub) TestStatic() { + // Static success + static := Static(s.Username, s.Password) + err := static(s.Username, s.Password) + assert.Nil(s.T(), err) + + // Static fail + static = Static(s.Username, s.Password) + err = static(s.Username, "wrong_pwd") + assert.NotNil(s.T(), err) +} + +func (s *SuiteGithub) TestGithubOrg() { + githubAuth := GithubOrg("github_org") + err := githubAuth("user", "token") + assert.NotNil(s.T(), err) +} + +func TestGithub(t *testing.T) { + suite.Run(t, new(SuiteGithub)) +}