Merge pull request #5 from opennaslab/rl-dev

feat: refactor api

README.md

@@ -4,23 +4,23 @@ Take you to the land of light, the city of freedom(A unified external service ma
 
 对于公网服务暴露，存在如下场景：
 无公网：
-1.本地安装frpc，本地frpc配置/长期运行
-2.服务器安装frps，或者再添加nginx，服务端配置/长期运行
-3.域名配置/长期运行
+1. 本地安装frpc，本地frpc配置/长期运行
+2. 服务器安装frps，或者再添加nginx，服务端配置/长期运行
+3. 域名配置/长期运行
 
 
 其他本地和服务端的业务配置和安装
 
 
 有公网：
-1.本地ngin反向代理配置
-2.无
-3.域名配置
+1. 本地ngin反向代理配置
+2. 无
+3. 域名配置
 
 对于步骤1，2，3；每个步骤都可以抽象化，定义独立的行为，抽象模型如下：
-1.本地配置
-2.服务端配置
-3.域名配置
+1. 本地配置
+2. 服务端配置
+3. 域名配置
 
 
 # demo

cmd/main.go

@@ -16,6 +16,15 @@ limitations under the License.
 
 package main
 
-func main() {
+import (
+	"k8s.io/klog"
+
+	"opennaslab.io/bifrost/pkg/server"
+)
 
+func main() {
+	router := server.NewServerRouter()
+	if err := router.Run(":8080"); err != nil {
+		klog.Errorf("run server failed:%v", err)
+	}
 }

go.mod

@@ -57,4 +57,5 @@ require (
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog v1.0.0 // indirect
 )

go.sum

@@ -43,6 +43,7 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
 github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
 github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
@@ -235,4 +236,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

pkg/api/types.go

@@ -21,15 +21,23 @@ type RemoteConfigDefinition ConfigStepDefinition
 type DNSConfigDefinition ConfigStepDefinition
 
 type ConfigStepDefinition struct {
-	Name        string          `json:"name"`
-	Description string          `json:"description"`
-	Image       string          `json:"image"`
-	Parameters  []StepParameter `json:"parameters"`
+	Name        string        `json:"name"`
+	Description string        `json:"description"`
+	Image       string        `json:"image"`
+	Parameters  StepParameter `json:"parameters"`
 }
 
 type StepParameter struct {
-	In  []interface{} `json:"in"`
-	Out []interface{} `json:"out"`
+	In  []Parameter `json:"in"`
+	Out []Parameter `json:"out"`
+}
+
+type Parameter struct {
+	Name        string      `json:"name"`
+	Description string      `json:"description"`
+	Type        string      `json:"type"`
+	Required    bool        `json:"required"`
+	Items       []Parameter `json:"items"`
 }
 
 type ConfigurationWorkflow struct {

pkg/registry/dns_step.go

@@ -15,3 +15,36 @@ limitations under the License.
 */
 
 package registry
+
+import (
+	"os"
+
+	"gopkg.in/yaml.v3"
+
+	"opennaslab.io/bifrost/pkg/api"
+)
+
+const (
+	DNSStepDir = "dns-step"
+)
+
+func ListAllDNSSteps(refresh bool) ([]api.DNSConfigDefinition, error) {
+	stepDir := RegistryCacheDir + "/" + DNSStepDir
+	files, err := CloneRegistry(refresh, RegistryCacheDir, stepDir)
+	if err != nil {
+		return nil, err
+	}
+
+	list := []api.DNSConfigDefinition{}
+	for _, file := range files {
+		data, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		def := api.DNSConfigDefinition{}
+		yaml.Unmarshal(data, &def)
+		list = append(list, def)
+	}
+
+	return list, nil
+}

pkg/registry/local_step.go

@@ -18,52 +18,37 @@ package registry
 
 import (
 	"os"
-	"path/filepath"
 
-	"github.com/go-git/go-git/v5"
 	"gopkg.in/yaml.v3"
+	"k8s.io/klog"
 
 	"opennaslab.io/bifrost/pkg/api"
 )
 
 const (
-	RegistryGitDir = "local_step"
+	LocalStepDir = "local-step"
 )
 
-func ListAllLocalConfigActions(refresh bool) ([]api.ConfigStepDefinition, error) {
-	cloneDir := os.Getenv("HOME") + "/" + RegistryGitDir
-	if refresh {
-		os.RemoveAll(cloneDir)
-	}
-	_, err := os.Stat(cloneDir)
-	if err != nil {
-		_, err := git.PlainClone(cloneDir, false, &git.CloneOptions{
-			URL:             RegistryGitRepo,
-			SingleBranch:    true,
-			InsecureSkipTLS: true,
-		})
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	var files []string
-	err = filepath.Walk(cloneDir, func(path string, info os.FileInfo, err error) error {
-		files = append(files, path)
-		return nil
-	})
+func ListAllLocalSteps(refresh bool) ([]api.LocalConfigDefinition, error) {
+	stepDir := RegistryCacheDir + "/" + LocalStepDir
+	files, err := CloneRegistry(refresh, RegistryCacheDir, stepDir)
 	if err != nil {
 		return nil, err
 	}
+	klog.Infof("jw1:%v", files)
 
-	list := []api.ConfigStepDefinition{}
+	list := []api.LocalConfigDefinition{}
 	for _, file := range files {
 		data, err := os.ReadFile(file)
 		if err != nil {
 			return nil, err
 		}
-		def := api.ConfigStepDefinition{}
-		yaml.Unmarshal(data, &def)
+		def := api.LocalConfigDefinition{}
+		klog.Infof("%v, %s", file, string(data))
+		if err := yaml.Unmarshal(data, &def); err != nil {
+			klog.Infof("jw5")
+			return nil, err
+		}
 		list = append(list, def)
 	}
 

pkg/registry/registry.go

@@ -16,7 +16,53 @@ limitations under the License.
 
 package registry
 
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/go-git/go-git/v5"
+	"k8s.io/klog"
+)
+
 const (
-	RegistryGitRepo  = "https://github.com/opennaslab/bifrost-registry"
-	RegistryCacheDir = "birefrost"
+	RegistryGitRepo = "https://github.com/opennaslab/bifrost-registry"
 )
+
+var (
+	RegistryCacheDir = os.Getenv("HOME") + "/birefrost-registry"
+)
+
+func CloneRegistry(refresh bool, homeDir, stepDir string) ([]string, error) {
+	klog.Infof("1")
+	if refresh {
+		os.RemoveAll(homeDir)
+	}
+	klog.Infof("2")
+	_, err := os.Stat(homeDir)
+	if err != nil {
+		klog.Infof("5")
+		_, err := git.PlainClone(homeDir, false, &git.CloneOptions{
+			URL:             RegistryGitRepo,
+			SingleBranch:    true,
+			InsecureSkipTLS: true,
+		})
+		if err != nil {
+			klog.Infof("3")
+			return nil, err
+		}
+	}
+
+	var files []string
+	err = filepath.Walk(stepDir, func(path string, info os.FileInfo, err error) error {
+		if path == stepDir {
+			return nil
+		}
+		files = append(files, path)
+		return nil
+	})
+	if err != nil {
+		klog.Infof("4")
+		return nil, err
+	}
+	return files, nil
+}

pkg/registry/remote_step.go

@@ -15,3 +15,36 @@ limitations under the License.
 */
 
 package registry
+
+import (
+	"os"
+
+	"gopkg.in/yaml.v3"
+
+	"opennaslab.io/bifrost/pkg/api"
+)
+
+const (
+	RemoteStepDir = "remote-step"
+)
+
+func ListAllRemoteSteps(refresh bool) ([]api.RemoteConfigDefinition, error) {
+	stepDir := RegistryCacheDir + "/" + RemoteStepDir
+	files, err := CloneRegistry(refresh, RegistryCacheDir, stepDir)
+	if err != nil {
+		return nil, err
+	}
+
+	list := []api.RemoteConfigDefinition{}
+	for _, file := range files {
+		data, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		def := api.RemoteConfigDefinition{}
+		yaml.Unmarshal(data, &def)
+		list = append(list, def)
+	}
+
+	return list, nil
+}

pkg/server/handler.go

@@ -17,11 +17,36 @@ limitations under the License.
 package server
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/gin-gonic/gin"
+	"k8s.io/klog"
+
+	"opennaslab.io/bifrost/pkg/registry"
 )
 
-func ListConfigHandler(ctx *gin.Context) {
+func ListLocalStepsHandler(ctx *gin.Context) {
+	refreshRegistry := ctx.Query("refresh") == "true"
+	if refreshRegistry {
+		ctx.JSON(http.StatusOK, []byte("{}"))
+	}
+	steps, err := registry.ListAllLocalSteps(refreshRegistry)
+	if err != nil {
+		klog.Errorf("List local steps failed:%v", err)
+		ctx.AbortWithError(http.StatusInternalServerError, err)
+		return
+	}
+	respData, err := json.Marshal(steps)
+	if err != nil {
+		klog.Errorf("Marshal local steps failed:%v", err)
+		ctx.AbortWithError(http.StatusInternalServerError, err)
+		return
+	}
+	klog.Infof("jw1:%s", respData)
+	ctx.JSON(http.StatusOK, respData)
+}
+
+func GetLocalStepHandler(ctx *gin.Context) {
 	ctx.Data(http.StatusOK, "application/json", []byte("OK"))
 }

pkg/server/router.go

@@ -30,37 +30,13 @@ func NewServerRouter() *gin.Engine {
 	corsHandler := cors.New(config)
 
 	initConfigRouter(router, corsHandler)
-	initActionRouter(router, corsHandler)
-	initWorkflowRouter(router, corsHandler)
 
 	return router
 }
 
 func initConfigRouter(router *gin.Engine, corsHandler gin.HandlerFunc) {
-	configGroup := router.Group("/api/v1/configs")
-	// List all configs
-	configGroup.GET("", ListConfigHandler)
-	// Get specific config
-	configGroup.GET("/:config_name", nil)
-	// Create or update a new config
-	configGroup.POST("/:config_name", nil)
-}
-
-func initActionRouter(router *gin.Engine, corsHandler gin.HandlerFunc) {
-	actionGroup := router.Group("/api/v1/actions")
-	// List all actions
-	actionGroup.GET("", nil)
-	// Get specific action
-	actionGroup.GET("/:action_name", nil)
-	// Create or update a new action
-}
-
-func initWorkflowRouter(router *gin.Engine, corsHandler gin.HandlerFunc) {
-	workflowGroup := router.Group("/api/v1/workflows")
-	// List all workflows
-	workflowGroup.GET("", nil)
-	// Get specific workflow
-	workflowGroup.GET("/:workflow_name", nil)
-	// Create or update a new workflow
-	workflowGroup.POST("/:workflow_name", nil)
+	configGroup := router.Group("/api/v1/localsteps")
+	configGroup.GET("", ListLocalStepsHandler)
+	// Get specific step
+	configGroup.GET("/:name", GetLocalStepHandler)
 }