diff --git a/.github/script/codearts_check.sh b/.github/script/codearts_check.sh new file mode 100644 index 0000000..b0b84e2 --- /dev/null +++ b/.github/script/codearts_check.sh @@ -0,0 +1,79 @@ + +IAM_DATA=$(cat < label.name); + if (!labels.includes(requiredLabel)) { + throw new Error(`PR 必须包含标签: ${requiredLabel}`); + } diff --git a/auth/gitee.go b/auth/gitee.go index 40d400a..003344b 100644 --- a/auth/gitee.go +++ b/auth/gitee.go @@ -15,6 +15,7 @@ import ( var ( clientId string clientSecret string + defaultToken string ) var ( @@ -23,6 +24,11 @@ var ( downloadPermissions = []string{"admin", "developer", "read"} ) +const contentType = "Content-Type" +const verifyLog = "verifyUser" +const appendPathAccessToken = "?access_token=" +const formatLogString = "%s | %s" + type giteeUser struct { Permission string `json:"permission"` } @@ -64,6 +70,13 @@ func Init(cfg *config.Config) error { return errors.New("client secret required") } } + defaultToken = cfg.DefaultToken + if defaultToken == "" { + defaultToken = os.Getenv("DEFAULT_TOKEN") + if defaultToken == "" { + return errors.New("default token required") + } + } return nil } @@ -90,14 +103,17 @@ func GiteeAuth() func(UserInRepo) error { // CheckRepoOwner checks whether the owner of a repo is allowed to use lfs server func CheckRepoOwner(userInRepo UserInRepo) error { path := fmt.Sprintf( - "https://gitee.com/api/v5/repos/%s/%s", + "https://gitee.com/api/v5/repos/%s/%s%s", userInRepo.Owner, userInRepo.Repo, + appendPathAccessToken, ) if userInRepo.Token != "" { - path += fmt.Sprintf("?access_token=%s", userInRepo.Token) + path += userInRepo.Token + } else { + path += defaultToken } - headers := http.Header{"Content-Type": []string{"application/json;charset=UTF-8"}} + headers := http.Header{contentType: []string{"application/json;charset=UTF-8"}} repo := new(Repo) err := getParsedResponse("GET", path, headers, nil, &repo) if err != nil { @@ -133,7 +149,7 @@ func getToken(username, password string) (string, error) { form.Add("client_secret", clientSecret) path := "https://gitee.com/oauth/token" - headers := http.Header{"Content-Type": []string{"application/x-www-form-urlencoded"}} + headers := http.Header{contentType: []string{"application/x-www-form-urlencoded"}} accessToken := new(AccessToken) err := getParsedResponse("POST", path, headers, strings.NewReader(form.Encode()), &accessToken) if err != nil { @@ -147,48 +163,59 @@ func getToken(username, password string) (string, error) { // verifyUser verifies user permission in repo by access_token and operation func verifyUser(userInRepo UserInRepo) error { path := fmt.Sprintf( - "https://gitee.com/api/v5/repos/%s/%s/collaborators/%s/permission", + "https://gitee.com/api/v5/repos/%s/%s/collaborators/%s/permission%s", userInRepo.Owner, userInRepo.Repo, userInRepo.Username, + appendPathAccessToken, ) if userInRepo.Token != "" { - path += fmt.Sprintf("?access_token=%s", userInRepo.Token) + path += userInRepo.Token + } else { + path += defaultToken } - headers := http.Header{"Content-Type": []string{"application/json;charset=UTF-8"}} + headers := http.Header{contentType: []string{"application/json;charset=UTF-8"}} giteeUser := new(giteeUser) err := getParsedResponse("GET", path, headers, nil, &giteeUser) if err != nil { msg := err.Error() + ": verify user permission failed" - logrus.Error(fmt.Sprintf("verifyUser | %s", msg)) + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) return errors.New(msg) } if userInRepo.Operation == "upload" { - for _, v := range uploadPermissions { - if giteeUser.Permission == v { - return nil - } - } - msg := fmt.Sprintf("forbidden: user %s has no permission to upload to %s/%s", - userInRepo.Username, userInRepo.Owner, userInRepo.Repo) - remindMsg := " \n如果您正在向fork仓库上传大文件，请确认您已使用如下命令修改了本地仓库的配置：" + - "\n`git config --local lfs.url https://artifacts.openeuler.openatom.cn/{owner}/{repo}`" + - "，\n其中{owner}/{repo}请改为您fork之后的仓库的名称" - logrus.Error(fmt.Sprintf("verifyUser | %s", msg)) - return errors.New(msg + remindMsg) + return verifyUserUpload(giteeUser, userInRepo) } else if userInRepo.Operation == "download" { - for _, v := range downloadPermissions { - if giteeUser.Permission == v { - return nil - } - } - msg := fmt.Sprintf("forbidden: user %s has no permission to download", userInRepo.Username) - logrus.Error(fmt.Sprintf("verifyUser | %s", msg)) - return errors.New(msg) + return verifyUserDownload(giteeUser, userInRepo) } else { msg := "system_error: unknow operation" - logrus.Error(fmt.Sprintf("verifyUser | %s", msg)) + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) return errors.New(msg) } } + +func verifyUserUpload(giteeUser *giteeUser, userInRepo UserInRepo) error { + for _, v := range uploadPermissions { + if giteeUser.Permission == v { + return nil + } + } + msg := fmt.Sprintf("forbidden: user %s has no permission to upload to %s/%s", + userInRepo.Username, userInRepo.Owner, userInRepo.Repo) + remindMsg := " \n如果您正在向fork仓库上传大文件，请确认您已使用如下命令修改了本地仓库的配置：" + + "\n`git config --local lfs.url https://artlfs.openeuler.openatom.cn/{owner}/{repo}`" + + "，\n其中{owner}/{repo}请改为您fork之后的仓库的名称" + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg + remindMsg) +} + +func verifyUserDownload(giteeUser *giteeUser, userInRepo UserInRepo) error { + for _, v := range downloadPermissions { + if giteeUser.Permission == v { + return nil + } + } + msg := fmt.Sprintf("forbidden: user %s has no permission to download", userInRepo.Username) + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg) +} diff --git a/auth/gitee_test.go b/auth/gitee_test.go index e5f085a..bc5c6fb 100644 --- a/auth/gitee_test.go +++ b/auth/gitee_test.go @@ -1,6 +1,7 @@ package auth import ( + "github.com/metalogical/BigFiles/config" "testing" "github.com/stretchr/testify/assert" @@ -10,23 +11,49 @@ import ( // SuiteGitee used for testing type SuiteGitee struct { suite.Suite - Repo string - Owner string + cfg config.Config + Repo string + Owner string + UserName string + Password string } // SetupSuite used for testing func (s *SuiteGitee) SetupSuite() { s.Repo = "software-package-server" s.Owner = "src-openeuler" + s.UserName = "user" + s.Password = "wrong_pwd" + s.cfg = config.Config{ + ClientId: "clientId", + ClientSecret: "clientSecret", + DefaultToken: "defaultToken", + } +} + +func (s *SuiteGitee) TestInit() { + //Init success + err := Init(&s.cfg) + assert.Nil(s.T(), err) } -// TearDownSuite used for testing -func (s *SuiteGitee) TearDownSuite() { +func (s *SuiteGitee) TestGiteeAuth() { + // GiteeAuth fail + userInRepo := UserInRepo{ + Repo: s.Repo, + Owner: s.Owner, + Username: s.UserName, + Password: s.Password, + Operation: "download", + } + giteeAuth := GiteeAuth() + err := giteeAuth(userInRepo) + assert.NotNil(s.T(), err) } func (s *SuiteGitee) TestGetToken() { // getToken fail - token, err := getToken("user", "wrong_pwd") + token, err := getToken(s.UserName, s.Password) assert.Equal(s.T(), "", token) assert.NotNil(s.T(), err.Error()) } @@ -36,14 +63,16 @@ func (s *SuiteGitee) TestCheckRepoOwner() { userInRepo := UserInRepo{ Repo: s.Repo, Owner: s.Owner, + Token: s.cfg.DefaultToken, } err := CheckRepoOwner(userInRepo) - assert.Nil(s.T(), err) + assert.NotNil(s.T(), err) // check no_exist repo userInRepo = UserInRepo{ Repo: "repo", Owner: "owner", + Token: s.cfg.DefaultToken, } err = CheckRepoOwner(userInRepo) assert.NotNil(s.T(), err) @@ -53,11 +82,17 @@ func (s *SuiteGitee) TestVerifyUser() { userInRepo := UserInRepo{ Repo: s.Repo, Owner: s.Owner, + Username: s.UserName, Operation: "download", + Token: s.cfg.DefaultToken, } err := verifyUser(userInRepo) assert.NotNil(s.T(), err) + + userInRepo.Operation = "upload" + err = verifyUser(userInRepo) + assert.NotNil(s.T(), err) } func TestGitee(t *testing.T) { diff --git a/auth/github_test.go b/auth/github_test.go index 1c65c84..f325e54 100644 --- a/auth/github_test.go +++ b/auth/github_test.go @@ -20,10 +20,6 @@ func (s *SuiteGithub) SetupSuite() { s.Password = "password" } -// TearDownSuite used for testing -func (s *SuiteGithub) TearDownSuite() { -} - func (s *SuiteGithub) TestStatic() { // Static success static := Static(s.Username, s.Password) diff --git a/config.yml.example b/config.yml.example index e7b0068..80c0f4d 100644 --- a/config.yml.example +++ b/config.yml.example @@ -1,8 +1,16 @@ -LFS_BUCKET: *********** -CDN_DOMAIN: *********** -OBS_REGION: *********** -OBS_ACCESS_KEY_ID: *********** -OBS_SECRET_ACCESS_KEY: *********** -CLIENT_ID: *********** -CLIENT_SECRET: *********** -PATH_PREFIX: *********** \ No newline at end of file +{ + "LFS_BUCKET": *********** + "CDN_DOMAIN": *********** + "OBS_REGION": *********** + "OBS_ACCESS_KEY_ID": *********** + "OBS_SECRET_ACCESS_KEY": *********** + "CLIENT_ID": *********** + "CLIENT_SECRET": *********** + "PATH_PREFIX": *********** + "VALIDATE_REGEXP": { + "OWNER_REGEXP": "^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$", + "REPONAME_REGEXP": "^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$", + "USERNAME_REGEXP": "^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$", + "PASSWORD_REGEXP": "^[a-zA-Z0-9!@_#$%^&*()\\-=+,?.,]*$" + } +} diff --git a/config/config.go b/config/config.go index 64ad6e6..adfa5d3 100644 --- a/config/config.go +++ b/config/config.go @@ -7,14 +7,23 @@ import ( ) type Config struct { - Prefix string `json:"PATH_PREFIX"` - LfsBucket string `json:"LFS_BUCKET"` - ClientId string `json:"CLIENT_ID"` - ClientSecret string `json:"CLIENT_SECRET"` - CdnDomain string `json:"CDN_DOMAIN"` - ObsRegion string `json:"OBS_REGION"` - ObsAccessKeyId string `json:"OBS_ACCESS_KEY_ID"` - ObsSecretAccessKey string `json:"OBS_SECRET_ACCESS_KEY"` + Prefix string `json:"PATH_PREFIX"` + LfsBucket string `json:"LFS_BUCKET"` + ClientId string `json:"CLIENT_ID"` + ClientSecret string `json:"CLIENT_SECRET"` + CdnDomain string `json:"CDN_DOMAIN"` + ObsRegion string `json:"OBS_REGION"` + ObsAccessKeyId string `json:"OBS_ACCESS_KEY_ID"` + ObsSecretAccessKey string `json:"OBS_SECRET_ACCESS_KEY"` + ValidateConfig ValidateConfig `json:"VALIDATE_REGEXP"` + DefaultToken string `json:"DEFAULT_TOKEN"` +} + +type ValidateConfig struct { + OwnerRegexp string `json:"OWNER_REGEXP" required:"true"` + RepoNameRegexp string `json:"REPONAME_REGEXP" required:"true"` + UsernameRegexp string `json:"USERNAME_REGEXP" required:"true"` + PasswordRegexp string `json:"PASSWORD_REGEXP" required:"true"` } // LoadConfig loads the configuration file from the specified path and deletes the file if needed diff --git a/main.go b/main.go index 95eeb3f..b8570a4 100644 --- a/main.go +++ b/main.go @@ -88,6 +88,12 @@ func main() { return } + if err := server.Init(cfg.ValidateConfig); err != nil { + logrus.Errorf("load ValidateConfig, err:%s", err.Error()) + + return + } + if err := auth.Init(cfg); err != nil { logrus.Errorf("load gitee config, err:%s", err.Error()) diff --git a/server/server.go b/server/server.go index 26d4548..a4dd565 100644 --- a/server/server.go +++ b/server/server.go @@ -21,6 +21,7 @@ import ( var ObsPutLimit int = 5*int(math.Pow10(9)) - 1 // 5GB - 1 var oidRegexp = regexp.MustCompile("^[a-f0-9]{64}$") +var contentType = "Content-Type" type Options struct { // required @@ -114,7 +115,7 @@ func (s *server) key(oid string) string { } func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/vnd.git-lfs+json") + w.Header().Set(contentType, "application/vnd.git-lfs+json") w.Header().Set("X-Content-Type-Options", "nosniff") var req batch.Request @@ -132,6 +133,15 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { userInRepo.Operation = req.Operation userInRepo.Owner = chi.URLParam(r, "owner") userInRepo.Repo = chi.URLParam(r, "repo") + + if !validatecfg.ownerRegexp.MatchString(userInRepo.Owner) || !validatecfg.reponameRegexp.MatchString(userInRepo.Repo) { + w.WriteHeader(http.StatusBadRequest) + must(json.NewEncoder(w).Encode(batch.ErrorResponse{ + Message: "invalid owner or reponame format", + })) + return + } + if err = auth.CheckRepoOwner(userInRepo); req.Operation == "upload" || err != nil { err := s.dealWithAuthError(userInRepo, w, r) if err != nil { @@ -160,6 +170,8 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { s.downloadObject(&in, out) case "upload": s.uploadObject(&in, out) + default: + continue } } must(json.NewEncoder(w).Encode(resp)) @@ -170,6 +182,15 @@ func (s *server) dealWithAuthError(userInRepo auth.UserInRepo, w http.ResponseWr if username, password, ok := r.BasicAuth(); ok { userInRepo.Username = username userInRepo.Password = password + + if !validatecfg.usernameRegexp.MatchString(userInRepo.Username) || + !validatecfg.passwordRegexp.MatchString(userInRepo.Password) { + w.WriteHeader(http.StatusBadRequest) + must(json.NewEncoder(w).Encode(batch.ErrorResponse{ + Message: "invalid username or password format", + })) + return errors.New("invalid username or password format") + } err = s.isAuthorized(userInRepo) } else { err = errors.New("unauthorized: cannot get password") @@ -190,6 +211,7 @@ func (s *server) dealWithAuthError(userInRepo auth.UserInRepo, w http.ResponseWr })) return err } + return nil } @@ -206,13 +228,15 @@ func (s *server) downloadObject(in *batch.RequestObject, out *batch.Object) { Code: 422, Message: "found object with wrong size", } + } else { + logrus.Infof("Metadata check pass, Size check pass") } getObjectInput := &obs.CreateSignedUrlInput{} getObjectInput.Method = obs.HttpMethodGet getObjectInput.Bucket = s.bucket getObjectInput.Key = s.key(in.OID) getObjectInput.Expires = int(s.ttl / time.Second) - getObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} + getObjectInput.Headers = map[string]string{contentType: "application/octet-stream"} // 生成下载对象的带授权信息的URL v := s.generateDownloadUrl(getObjectInput) @@ -246,7 +270,7 @@ func (s *server) uploadObject(in *batch.RequestObject, out *batch.Object) { putObjectInput.Bucket = s.bucket putObjectInput.Key = s.key(in.OID) putObjectInput.Expires = int(s.ttl / time.Second) - putObjectInput.Headers = map[string]string{"Content-Type": "application/octet-stream"} + putObjectInput.Headers = map[string]string{contentType: "application/octet-stream"} putObjectOutput, err := s.client.CreateSignedUrl(putObjectInput) if err != nil { panic(err) @@ -292,7 +316,7 @@ func (s *server) healthCheck(w http.ResponseWriter, r *http.Request) { Data: "healthCheck success", } - w.Header().Set("Content-Type", "application/json") + w.Header().Set(contentType, "application/json") w.WriteHeader(http.StatusOK) must(json.NewEncoder(w).Encode(response)) } diff --git a/server/validate.go b/server/validate.go new file mode 100644 index 0000000..f7e40fb --- /dev/null +++ b/server/validate.go @@ -0,0 +1,41 @@ +package server + +import ( + "fmt" + "github.com/metalogical/BigFiles/config" + "regexp" +) + +type validateConfig struct { + ownerRegexp *regexp.Regexp + reponameRegexp *regexp.Regexp + usernameRegexp *regexp.Regexp + passwordRegexp *regexp.Regexp +} + +var validatecfg validateConfig + +func Init(cfg config.ValidateConfig) error { + var err error + validatecfg.ownerRegexp, err = regexp.Compile(cfg.OwnerRegexp) + if err != nil { + return fmt.Errorf("failed to compile owner regexp: %w", err) + } + + validatecfg.reponameRegexp, err = regexp.Compile(cfg.RepoNameRegexp) + if err != nil { + return fmt.Errorf("failed to compile repo name regexp: %w", err) + } + + validatecfg.usernameRegexp, err = regexp.Compile(cfg.UsernameRegexp) + if err != nil { + return fmt.Errorf("failed to compile username regexp: %w", err) + } + + validatecfg.passwordRegexp, err = regexp.Compile(cfg.PasswordRegexp) + if err != nil { + return fmt.Errorf("failed to compile password regexp: %w", err) + } + + return nil +}