diff --git a/.gitignore b/.gitignore index 58e32972..1a83ce0c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ ops/testnet/.terraform/providers .vscode/ .run/ bin/ -vendor/ \ No newline at end of file +vendor/ +./data/ \ No newline at end of file diff --git a/pkg/module/shortcuts/shortcuts.go b/pkg/module/shortcuts/shortcuts.go index e8d9191b..71693c28 100644 --- a/pkg/module/shortcuts/shortcuts.go +++ b/pkg/module/shortcuts/shortcuts.go @@ -13,6 +13,8 @@ import ( // parse something with no slashes in it as // github.com/bacalhau-project/lilypad-module- +const LILYPAD_MODULE_CONFIG_PATH = "/lilypad_module.json.tmpl" + func GetModule(name string) (data.ModuleConfig, error) { // parse name per following valid formats // github.com/user/repo:tag --> Repo: https://github.com/user/repo; Hash = tag @@ -20,24 +22,25 @@ func GetModule(name string) (data.ModuleConfig, error) { if name == "" { return data.ModuleConfig{}, fmt.Errorf("module name is empty") } - var repo, hash string parts := strings.Split(name, ":") if len(parts) != 2 { return data.ModuleConfig{}, fmt.Errorf("invalid module name format: %s", name) } - hash = parts[1] + repo, hash := parts[0], parts[1] if strings.Contains(name, "/") { - repo = fmt.Sprintf("https://%s", parts[0]) + // 3rd party module + repo = fmt.Sprintf("https://%s", repo) } else { - repo = fmt.Sprintf("https://github.com/bacalhau-project/lilypad-module-%s", parts[0]) + // lilypad std module + repo = fmt.Sprintf("https://github.com/bacalhau-project/lilypad-module-%s", repo) } // TODO: docs for authoring a module module := data.ModuleConfig{ - Name: "", + Name: "", // TODO: Repo: repo, Hash: hash, - Path: "/lilypad_module.json.tmpl", + Path: LILYPAD_MODULE_CONFIG_PATH, } return module, nil diff --git a/pkg/module/utils.go b/pkg/module/utils.go index 0ef86570..d72f8983 100644 --- a/pkg/module/utils.go +++ b/pkg/module/utils.go @@ -2,6 +2,7 @@ package module import ( "bytes" + "context" "encoding/json" "fmt" "net/url" @@ -24,7 +25,7 @@ const REPO_DIR = "repos" func getRepoLocalPath(repoURL string) (string, error) { parsedURL, err := url.Parse(repoURL) if err != nil { - return "", err + return "", fmt.Errorf("url parsing failed with %v", err) } pathParts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/") @@ -71,7 +72,7 @@ func ProcessModule(module data.ModuleConfig) (data.ModuleConfig, error) { // TODO: check if we have the repo already cloned // handle fetching new changes perhaps the commit hash is not the latest // at the moment we will do the slow thing and clone the repo every time -func CloneModule(module data.ModuleConfig) (*git.Repository, error) { +func CloneModule(module data.ModuleConfig) (repo *git.Repository, err error) { repoPath, err := getRepoLocalPath(module.Repo) if err != nil { return nil, err @@ -81,47 +82,61 @@ func CloneModule(module data.ModuleConfig) (*git.Repository, error) { return nil, err } fileInfo, err := os.Stat(filepath.Join(repoDir, ".git")) - if err == nil && fileInfo.IsDir() { - repo, err := git.PlainOpen(repoDir) - // err := os.RemoveAll(repoDir) - if err != nil { - return nil, err - } - // Check if hash or tag specified exists - h, err := repo.ResolveRevision(plumbing.Revision(module.Hash)) - if err != nil { - return nil, err - } - // XXX SECURITY: on RP side, need to verify this hash is in the allowlist - // explicitly to ensure determinism (and that we're running the code we - // explicitly approved) - _, err = repo.Storer.EncodedObject(plumbing.AnyObject, *h) - if err != nil { - // this means there is no hash in the repo - // so let's clean it up and clone it again - err = os.RemoveAll(repoDir) - if err != nil { - return nil, err - } - } else { - log.Debug(). - Str("repo exists", repoDir). - Str("repo remote", module.Repo). - Msgf("") - // this means we do have the hash locally so can just return the repo as is - return repo, nil - } + + repoCloned := err == nil && fileInfo.IsDir() + + if !repoCloned { + log.Debug(). + Str("repo clone", repoDir). + Str("repo remote", module.Repo). + Msgf("") + return git.PlainClone(repoDir, false, &git.CloneOptions{ + URL: module.Repo, + Progress: os.Stdout, + }) } + log.Debug(). - Str("repo clone", repoDir). + Str("repo exists", repoDir). Str("repo remote", module.Repo). Msgf("") - return git.PlainClone(repoDir, false, &git.CloneOptions{ - URL: module.Repo, - }) + + repo, err = git.PlainOpen(repoDir) + // err := os.RemoveAll(repoDir) + if err != nil { + return nil, err + } + + // git fetch origin: Resolves #https://github.com/bacalhau-project/lilypad/issues/13 + gitFetchOptions := &git.FetchOptions{ + Tags: git.AllTags, + Progress: os.Stdout, + } + gitFetchOptions.Validate() // sets default values like remote=origin + log.Info().Str("updating cached git repo", repoDir).Msgf("") + err = repo.FetchContext(context.Background(), gitFetchOptions) + + // Check if hash or tag specified exists + h, err := repo.ResolveRevision(plumbing.Revision(module.Hash)) + if err != nil { + return nil, err + } + // XXX SECURITY: on RP side, need to verify this hash is in the allowlist + // explicitly to ensure determinism (and that we're running the code we + // explicitly approved) + _, err = repo.Storer.EncodedObject(plumbing.AnyObject, *h) + if err != nil { + // this means there is no hash in the repo + // so let's clean it up and clone it again + err = os.RemoveAll(repoDir) + if err != nil { + return nil, err + } + } + return } -// get a module cloned and checked out then return the text content of the template +// PrepareModule get a module cloned and checked out then return the text content of the template // - process shortcuts // - check if we have the repo cloned // - checkout the correct hash diff --git a/pkg/system/fs.go b/pkg/system/fs.go index cec06a05..3bdeec71 100644 --- a/pkg/system/fs.go +++ b/pkg/system/fs.go @@ -11,6 +11,7 @@ import ( func dataDirPath(path string) string { basePath := os.Getenv("DATA_DIR") if basePath == "" { + // TODO: configure temp dir based on OS basePath = "/tmp/lilypad/data" } return filepath.Join(basePath, path)