fix: use GH token for VCS lookups (#578)

Signed-off-by: Grant Linville <[email protected]>

.github/workflows/smoke.yaml

@@ -139,8 +139,9 @@ jobs:
           go-version: "1.21"
       - env:
           OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
-          GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider@tool-beta
+          GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider
           ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
+          GPTSCRIPT_CREDENTIAL_OVERRIDE: "github.com/gptscript-ai/claude3-anthropic-provider/credential:ANTHROPIC_API_KEY"
         name: Run smoke test for claude-3-opus-20240229
         run: |
           echo "Running smoke test for model claude-3-opus-20240229"

pkg/loader/github/github.go

@@ -88,9 +88,9 @@ func getCommit(ctx context.Context, account, repo, ref string) (string, error) {
 	return commit.SHA, nil
 }
 
-func Load(ctx context.Context, _ *cache.Client, urlName string) (string, *types.Repo, bool, error) {
+func Load(ctx context.Context, _ *cache.Client, urlName string) (string, string, *types.Repo, bool, error) {
 	if !strings.HasPrefix(urlName, GithubPrefix) {
-		return "", nil, false, nil
+		return "", "", nil, false, nil
 	}
 
 	url, ref, _ := strings.Cut(urlName, "@")
@@ -101,15 +101,15 @@ func Load(ctx context.Context, _ *cache.Client, urlName string) (string, *types.
 	parts := strings.Split(url, "/")
 	// Must be at least 3 parts github.com/ACCOUNT/REPO[/FILE]
 	if len(parts) < 3 {
-		return "", nil, false, nil
+		return "", "", nil, false, nil
 	}
 
 	account, repo := parts[1], parts[2]
 	path := strings.Join(parts[3:], "/")
 
 	ref, err := getCommit(ctx, account, repo, ref)
 	if err != nil {
-		return "", nil, false, err
+		return "", "", nil, false, err
 	}
 
 	downloadURL := fmt.Sprintf(githubDownloadURL, account, repo, ref, path)
@@ -141,7 +141,7 @@ func Load(ctx context.Context, _ *cache.Client, urlName string) (string, *types.
 		path = testPath
 	}
 
-	return downloadURL, &types.Repo{
+	return downloadURL, githubAuthToken, &types.Repo{
 		VCS:      "git",
 		Root:     fmt.Sprintf(githubRepoURL, account, repo),
 		Path:     gpath.Dir(path),

pkg/loader/github/github_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestLoad(t *testing.T) {
-	url, repo, ok, err := Load(context.Background(), nil, "github.com/gptscript-ai/gptscript/pkg/loader/testdata/tool@172dfb0")
+	url, _, repo, ok, err := Load(context.Background(), nil, "github.com/gptscript-ai/gptscript/pkg/loader/testdata/tool@172dfb0")
 	require.NoError(t, err)
 	assert.True(t, ok)
 	autogold.Expect("https://raw.githubusercontent.com/gptscript-ai/gptscript/172dfb00b48c6adbbaa7e99270933f95887d1b91/pkg/loader/testdata/tool/tool.gpt").Equal(t, url)
@@ -22,7 +22,7 @@ func TestLoad(t *testing.T) {
 		Revision: "172dfb00b48c6adbbaa7e99270933f95887d1b91",
 	}).Equal(t, repo)
 
-	url, repo, ok, err = Load(context.Background(), nil, "github.com/gptscript-ai/gptscript/pkg/loader/testdata/agent@172dfb0")
+	url, _, repo, ok, err = Load(context.Background(), nil, "github.com/gptscript-ai/gptscript/pkg/loader/testdata/agent@172dfb0")
 	require.NoError(t, err)
 	assert.True(t, ok)
 	autogold.Expect("https://raw.githubusercontent.com/gptscript-ai/gptscript/172dfb00b48c6adbbaa7e99270933f95887d1b91/pkg/loader/testdata/agent/agent.gpt").Equal(t, url)
@@ -33,7 +33,7 @@ func TestLoad(t *testing.T) {
 		Revision: "172dfb00b48c6adbbaa7e99270933f95887d1b91",
 	}).Equal(t, repo)
 
-	url, repo, ok, err = Load(context.Background(), nil, "github.com/gptscript-ai/gptscript/pkg/loader/testdata/bothtoolagent@172dfb0")
+	url, _, repo, ok, err = Load(context.Background(), nil, "github.com/gptscript-ai/gptscript/pkg/loader/testdata/bothtoolagent@172dfb0")
 	require.NoError(t, err)
 	assert.True(t, ok)
 	autogold.Expect("https://raw.githubusercontent.com/gptscript-ai/gptscript/172dfb00b48c6adbbaa7e99270933f95887d1b91/pkg/loader/testdata/bothtoolagent/agent.gpt").Equal(t, url)

pkg/loader/url.go

@@ -14,7 +14,7 @@ import (
 	"github.com/gptscript-ai/gptscript/pkg/types"
 )
 
-type VCSLookup func(context.Context, *cache.Client, string) (string, *types.Repo, bool, error)
+type VCSLookup func(context.Context, *cache.Client, string) (string, string, *types.Repo, bool, error)
 
 var vcsLookups []VCSLookup
 
@@ -35,10 +35,11 @@ type cacheValue struct {
 
 func loadURL(ctx context.Context, cache *cache.Client, base *source, name string) (*source, bool, error) {
 	var (
-		repo      *types.Repo
-		url       = name
-		relative  = strings.HasPrefix(name, ".") || !strings.Contains(name, "/")
-		cachedKey = cacheKey{
+		repo        *types.Repo
+		url         = name
+		bearerToken = ""
+		relative    = strings.HasPrefix(name, ".") || !strings.Contains(name, "/")
+		cachedKey   = cacheKey{
 			Name: name,
 			Path: base.Path,
 			Repo: base.Repo,
@@ -67,12 +68,13 @@ func loadURL(ctx context.Context, cache *cache.Client, base *source, name string
 
 	if repo == nil || !relative {
 		for _, vcs := range vcsLookups {
-			newURL, newRepo, ok, err := vcs(ctx, cache, name)
+			newURL, newBearer, newRepo, ok, err := vcs(ctx, cache, name)
 			if err != nil {
 				return nil, false, err
 			} else if ok {
 				repo = newRepo
 				url = newURL
+				bearerToken = newBearer
 				break
 			}
 		}
@@ -105,6 +107,10 @@ func loadURL(ctx context.Context, cache *cache.Client, base *source, name string
 		return nil, false, err
 	}
 
+	if bearerToken != "" {
+		req.Header.Set("Authorization", "Bearer "+bearerToken)
+	}
+
 	data, err := getWithDefaults(req)
 	if err != nil {
 		return nil, false, fmt.Errorf("error loading %s: %v", url, err)

pkg/repos/get.go

@@ -128,7 +128,7 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co
 	}
 
 	// Load the credential helpers repo information.
-	_, repo, _, err := github.Load(ctx, nil, credentialHelpersRepo)
+	_, _, repo, _, err := github.Load(ctx, nil, credentialHelpersRepo)
 	if err != nil {
 		return err
 	}