Merge pull request #19 from bacalhau-project/bugfix/github-tags-not-f…

…ound

hotfix: update cached git repo

.gitignore

@@ -12,4 +12,5 @@ ops/testnet/.terraform/providers
 .vscode/
 .run/
 bin/
-vendor/
+vendor/
+./data/

pkg/module/shortcuts/shortcuts.go

@@ -13,31 +13,34 @@ import (
 // parse something with no slashes in it as
 // github.com/bacalhau-project/lilypad-module-<shortcode>
 
+const LILYPAD_MODULE_CONFIG_PATH = "/lilypad_module.json.tmpl"
+
 func GetModule(name string) (data.ModuleConfig, error) {
 	// parse name per following valid formats
 	// github.com/user/repo:tag --> Repo: https://github.com/user/repo; Hash = tag
 	// bar:tag --> Repo = https://github.com/bacalhau-project/lilypad-module-<bar>, Hash = tag
 	if name == "" {
 		return data.ModuleConfig{}, fmt.Errorf("module name is empty")
 	}
-	var repo, hash string
 	parts := strings.Split(name, ":")
 	if len(parts) != 2 {
 		return data.ModuleConfig{}, fmt.Errorf("invalid module name format: %s", name)
 	}
-	hash = parts[1]
+	repo, hash := parts[0], parts[1]
 	if strings.Contains(name, "/") {
-		repo = fmt.Sprintf("https://%s", parts[0])
+		// 3rd party module
+		repo = fmt.Sprintf("https://%s", repo)
 	} else {
-		repo = fmt.Sprintf("https://github.com/bacalhau-project/lilypad-module-%s", parts[0])
+		// lilypad std module
+		repo = fmt.Sprintf("https://github.com/bacalhau-project/lilypad-module-%s", repo)
 	}
 
 	// TODO: docs for authoring a module
 	module := data.ModuleConfig{
-		Name: "",
+		Name: "", // TODO:
 		Repo: repo,
 		Hash: hash,
-		Path: "/lilypad_module.json.tmpl",
+		Path: LILYPAD_MODULE_CONFIG_PATH,
 	}
 
 	return module, nil

pkg/module/utils.go

@@ -2,6 +2,7 @@ package module
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/url"
@@ -24,7 +25,7 @@ const REPO_DIR = "repos"
 func getRepoLocalPath(repoURL string) (string, error) {
 	parsedURL, err := url.Parse(repoURL)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("url parsing failed with %v", err)
 	}
 
 	pathParts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/")
@@ -71,7 +72,7 @@ func ProcessModule(module data.ModuleConfig) (data.ModuleConfig, error) {
 // TODO: check if we have the repo already cloned
 // handle fetching new changes perhaps the commit hash is not the latest
 // at the moment we will do the slow thing and clone the repo every time
-func CloneModule(module data.ModuleConfig) (*git.Repository, error) {
+func CloneModule(module data.ModuleConfig) (repo *git.Repository, err error) {
 	repoPath, err := getRepoLocalPath(module.Repo)
 	if err != nil {
 		return nil, err
@@ -81,47 +82,61 @@ func CloneModule(module data.ModuleConfig) (*git.Repository, error) {
 		return nil, err
 	}
 	fileInfo, err := os.Stat(filepath.Join(repoDir, ".git"))
-	if err == nil && fileInfo.IsDir() {
-		repo, err := git.PlainOpen(repoDir)
-		// err := os.RemoveAll(repoDir)
-		if err != nil {
-			return nil, err
-		}
-		// Check if hash or tag specified exists
-		h, err := repo.ResolveRevision(plumbing.Revision(module.Hash))
-		if err != nil {
-			return nil, err
-		}
-		// XXX SECURITY: on RP side, need to verify this hash is in the allowlist
-		// explicitly to ensure determinism (and that we're running the code we
-		// explicitly approved)
-		_, err = repo.Storer.EncodedObject(plumbing.AnyObject, *h)
-		if err != nil {
-			// this means there is no hash in the repo
-			// so let's clean it up and clone it again
-			err = os.RemoveAll(repoDir)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			log.Debug().
-				Str("repo exists", repoDir).
-				Str("repo remote", module.Repo).
-				Msgf("")
-			// this means we do have the hash locally so can just return the repo as is
-			return repo, nil
-		}
+
+	repoCloned := err == nil && fileInfo.IsDir()
+
+	if !repoCloned {
+		log.Debug().
+			Str("repo clone", repoDir).
+			Str("repo remote", module.Repo).
+			Msgf("")
+		return git.PlainClone(repoDir, false, &git.CloneOptions{
+			URL:      module.Repo,
+			Progress: os.Stdout,
+		})
 	}
+
 	log.Debug().
-		Str("repo clone", repoDir).
+		Str("repo exists", repoDir).
 		Str("repo remote", module.Repo).
 		Msgf("")
-	return git.PlainClone(repoDir, false, &git.CloneOptions{
-		URL: module.Repo,
-	})
+
+	repo, err = git.PlainOpen(repoDir)
+	// err := os.RemoveAll(repoDir)
+	if err != nil {
+		return nil, err
+	}
+
+	// git fetch origin: Resolves #https://github.com/bacalhau-project/lilypad/issues/13
+	gitFetchOptions := &git.FetchOptions{
+		Tags:     git.AllTags,
+		Progress: os.Stdout,
+	}
+	gitFetchOptions.Validate() // sets default values like remote=origin
+	log.Info().Str("updating cached git repo", repoDir).Msgf("")
+	err = repo.FetchContext(context.Background(), gitFetchOptions)
+
+	// Check if hash or tag specified exists
+	h, err := repo.ResolveRevision(plumbing.Revision(module.Hash))
+	if err != nil {
+		return nil, err
+	}
+	// XXX SECURITY: on RP side, need to verify this hash is in the allowlist
+	// explicitly to ensure determinism (and that we're running the code we
+	// explicitly approved)
+	_, err = repo.Storer.EncodedObject(plumbing.AnyObject, *h)
+	if err != nil {
+		// this means there is no hash in the repo
+		// so let's clean it up and clone it again
+		err = os.RemoveAll(repoDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return
 }
 
-// get a module cloned and checked out then return the text content of the template
+// PrepareModule get a module cloned and checked out then return the text content of the template
 //   - process shortcuts
 //   - check if we have the repo cloned
 //   - checkout the correct hash

pkg/system/fs.go

@@ -11,6 +11,7 @@ import (
 func dataDirPath(path string) string {
 	basePath := os.Getenv("DATA_DIR")
 	if basePath == "" {
+		// TODO: configure temp dir based on OS
 		basePath = "/tmp/lilypad/data"
 	}
 	return filepath.Join(basePath, path)