From 9b2d6dc3ed58c6bab5fcf35568aeb951cc2d9b87 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Fri, 7 Jun 2024 10:43:43 -0400 Subject: [PATCH] set up cred helpers Signed-off-by: Grant Linville --- docs/docs/02-credentials.md | 71 ++++++++++++++++ ...-credentials.md => 04-credential-tools.md} | 24 +----- pkg/cli/credential.go | 9 +- pkg/cli/credential_delete.go | 9 +- pkg/config/cliconfig.go | 60 +++++++++++--- pkg/credentials/store.go | 20 +++-- pkg/credentials/util.go | 17 ++++ pkg/engine/engine.go | 2 + pkg/gptscript/gptscript.go | 24 ++++-- pkg/openai/client.go | 18 ++-- pkg/prompt/credential.go | 12 +-- pkg/remote/remote.go | 22 +++-- pkg/repos/get.go | 82 +++++++++++++++++-- pkg/repos/runtimes/golang/golang.go | 27 ++++++ pkg/runner/runner.go | 29 +++---- pkg/tests/tester/runner.go | 2 +- 16 files changed, 322 insertions(+), 106 deletions(-) create mode 100644 docs/docs/02-credentials.md rename docs/docs/03-tools/{04-credentials.md => 04-credential-tools.md} (84%) create mode 100644 pkg/credentials/util.go diff --git a/docs/docs/02-credentials.md b/docs/docs/02-credentials.md new file mode 100644 index 00000000..fb62690c --- /dev/null +++ b/docs/docs/02-credentials.md @@ -0,0 +1,71 @@ +# Credentials + +Some GPTScript tools will use [credential tools](03-tools/04-credential-tools.md) to get sensitive information like API keys from the user. +These credentials will be stored in a credential store and are used to set environment variables before executing a tool. + +GPTScript itself will also prompt you for your OpenAI API key and save it in the credential store if the +`OPENAI_API_KEY` environment variable is not set. The environment variable always overrides the value stored in the +credential store. + +## Credential Store + +There are different options available for credential stores, depending on your operating system. +When you first run GPTScript, the default credential store for your operating system will be selected. + +You can change the credential store by modifying the `credsStore` field in your GPTScript configuration file. +The configuration file is located in the following location based on your operating system: +- Windows: `%APPDATA%\Local\gptscript\config.json` +- macOS: `$HOME/Library/Application Support/gptscript/config.json` +- Linux: `$XDG_CONFIG_HOME/gptscript/config.json` + +(Note: if you set the `XDG_CONFIG_HOME` environment variable on macOS, then the same path as Linux will be used.) + +The configured credential store will be automatically downloaded and compiled from the [gptscript-ai/gptscript-credential-helpers](https://github.com/gptscript-ai/gptscript-credential-helpers) +repository, other than the `file` store, which is built-in to GPTScript itself. +The `wincred` and `osxkeychain` stores do not require any external dependencies in order to compile correctly. +The `secretservice` store on Linux may require some extra packages to be installed, depending on your distribution. + +## Credential Store Options + +### Wincred (Windows) + +Wincred, or the Windows Credential Manager, is the default credential store for Windows. +This is Windows' built-in credential manager that securely stores credentials for Windows applications. +This credential store is called `wincred` in GPTScript's configuration. + +### macOS Keychain (macOS) + +The macOS Keychain is the default credential store for macOS. +This is macOS' built-in password manager that securely stores credentials for macOS applications. +This credential store is called `osxkeychain` in GPTScript's configuration. + +### File (all operating systems) + +"File" is the default credential store for every other operating system besides Windows and macOS, but it +can also be configured on Windows and macOS. This will store credentials **unencrypted** inside GPTScript's +configuration file. +This credential store is called `file` in GPTScript's configuration. + +### D-Bus Secret Service (Linux) + +The D-Bus Secret Service can be used as the credential store for Linux systems with a desktop environment that supports it. +This credential store is called `secretservice` in GPTScript's configuration. + +### Pass (Linux) + +Pass can be used as the credential store for Linux systems. This requires the `pass` package to be installed +and configured. See [this guide](https://www.howtogeek.com/devops/how-to-use-pass-a-command-line-password-manager-for-linux-systems/) +for information about how to set it up. +This credential store is called `pass` in GPTScript's configuration. + +## GPTScript `credential` Command + +The `gptscript credential` command can be used to interact with your stored credentials. +`gptscript credential` without any arguments will list all stored credentials. +`gptscript credential delete ` will delete the specified credential, and you will be +prompted to enter it again the next time a tool that requires it is run. + +## See Also + +For more advanced credential usage, including credential contexts, writing credential tools, and using +credential tools, see [the credential tools documentation](03-tools/04-credential-tools). diff --git a/docs/docs/03-tools/04-credentials.md b/docs/docs/03-tools/04-credential-tools.md similarity index 84% rename from docs/docs/03-tools/04-credentials.md rename to docs/docs/03-tools/04-credential-tools.md index a984db8e..ec8a6ad0 100644 --- a/docs/docs/03-tools/04-credentials.md +++ b/docs/docs/03-tools/04-credential-tools.md @@ -1,4 +1,4 @@ -# Credentials +# Credential Tools GPTScript supports credential provider tools. These tools can be used to fetch credentials from a secure location (or directly from user input) and conveniently set them in the environment before running a script. @@ -76,24 +76,8 @@ In this example, the tool's output would be `{"env":{"MY_ENV_VAR":"my value"}}` ## Storing Credentials -By default, credentials are automatically stored in a config file at `$XDG_CONFIG_HOME/gptscript/config.json`. -This config file also has another parameter, `credsStore`, which indicates where the credentials are being stored. - -- `file` (default): The credentials are stored directly in the config file. -- `osxkeychain`: The credentials are stored in the macOS Keychain. -- `wincred`: The credentials are stored in the Windows Credential Manager. - -In order to use `osxkeychain` or `wincred` as the credsStore, you must have the `gptscript-credential-*` executable -available in your PATH. There will probably be better packaging for this in the future, but for now, you can build them -from the [repo](https://github.com/gptscript-ai/gptscript-credential-helpers). (For wincred, make sure the executable -is called `gptscript-credential-wincred.exe`.) - -There will likely be support added for other credential stores in the future. - -:::note -Credentials received from credential provider tools that are not on GitHub (such as a local file) and do not have an alias -will not be stored in the credentials store. -::: +By default, credentials are automatically stored in the credential store. Read the [main credentials page](../02-credentials.md) +for more information about the credential store. ## Credential Aliases @@ -112,7 +96,7 @@ or when you want to store credentials that were provided by a tool that is not o ## Credential Contexts Each stored credential is uniquely identified by the name of its provider tool (or alias, if one was specified) and the name of its context. -A credential context is basically a namespace for credentials. If you have multiple credentials from the same provider tool, +A credential context is basically a namespace for credentials. If you have multiple credentials from the same provider tool, you can switch between them by defining them in different credential contexts. The default context is called `default`, and this is used if none is specified. diff --git a/pkg/cli/credential.go b/pkg/cli/credential.go index 52115b3b..cb0e698b 100644 --- a/pkg/cli/credential.go +++ b/pkg/cli/credential.go @@ -8,6 +8,7 @@ import ( "text/tabwriter" cmd2 "github.com/acorn-io/cmd" + "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/spf13/cobra" @@ -38,7 +39,13 @@ func (c *Credential) Run(_ *cobra.Command, _ []string) error { ctx = "*" } - store, err := credentials.NewStore(cfg, ctx) + opts, err := c.root.NewGPTScriptOpts() + if err != nil { + return err + } + opts.Cache = cache.Complete(opts.Cache) + + store, err := credentials.NewStore(cfg, ctx, opts.Cache.CacheDir) if err != nil { return fmt.Errorf("failed to get credentials store: %w", err) } diff --git a/pkg/cli/credential_delete.go b/pkg/cli/credential_delete.go index 59c61521..26d24b2a 100644 --- a/pkg/cli/credential_delete.go +++ b/pkg/cli/credential_delete.go @@ -3,6 +3,7 @@ package cli import ( "fmt" + "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/spf13/cobra" @@ -21,12 +22,18 @@ func (c *Delete) Customize(cmd *cobra.Command) { } func (c *Delete) Run(_ *cobra.Command, args []string) error { + opts, err := c.root.NewGPTScriptOpts() + if err != nil { + return err + } + opts.Cache = cache.Complete(opts.Cache) + cfg, err := config.ReadCLIConfig(c.root.ConfigFile) if err != nil { return fmt.Errorf("failed to read CLI config: %w", err) } - store, err := credentials.NewStore(cfg, c.root.CredentialContext) + store, err := credentials.NewStore(cfg, c.root.CredentialContext, opts.Cache.CacheDir) if err != nil { return fmt.Errorf("failed to get credentials store: %w", err) } diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index 54ab0c3e..e4aa49ab 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -3,10 +3,11 @@ package config import ( "encoding/base64" "encoding/json" + "errors" "fmt" "os" - "os/exec" "runtime" + "slices" "strings" "sync" @@ -14,6 +15,12 @@ import ( "github.com/docker/cli/cli/config/types" ) +var ( + darwinHelpers = []string{"osxkeychain", "file"} + windowsHelpers = []string{"wincred", "file"} + linuxHelpers = []string{"secretservice", "pass", "file"} +) + const GPTScriptHelperPrefix = "gptscript-credential-" type AuthConfig types.AuthConfig @@ -132,21 +139,54 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { } if result.CredentialsStore == "" { - result.setDefaultCredentialsStore() + if err := result.setDefaultCredentialsStore(); err != nil { + return nil, err + } + } + + if !isValidCredentialHelper(result.CredentialsStore) { + errMsg := fmt.Sprintf("invalid credential store '%s'", result.CredentialsStore) + switch runtime.GOOS { + case "darwin": + errMsg += " (use 'osxkeychain' or 'file')" + case "windows": + errMsg += " (use 'wincred' or 'file')" + case "linux": + errMsg += " (use 'secretservice', 'pass', or 'file')" + default: + errMsg += " (use 'file')" + } + errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.GPTScriptConfigFile) + + return nil, errors.New(errMsg) } return result, nil } -func (c *CLIConfig) setDefaultCredentialsStore() { - if runtime.GOOS == "darwin" { - // Check for the existence of the helper program - fullPath, err := exec.LookPath(GPTScriptHelperPrefix + "osxkeychain") - if err == nil && fullPath != "" { - c.CredentialsStore = "osxkeychain" - } +func (c *CLIConfig) setDefaultCredentialsStore() error { + switch runtime.GOOS { + case "darwin": + c.CredentialsStore = "osxkeychain" + case "windows": + c.CredentialsStore = "wincred" + default: + c.CredentialsStore = "file" + } + return c.Save() +} + +func isValidCredentialHelper(helper string) bool { + switch runtime.GOOS { + case "darwin": + return slices.Contains(darwinHelpers, helper) + case "windows": + return slices.Contains(windowsHelpers, helper) + case "linux": + return slices.Contains(linuxHelpers, helper) + default: + return helper == "file" } - c.CredentialsStore = "file" } func readFile(path string) ([]byte, error) { diff --git a/pkg/credentials/store.go b/pkg/credentials/store.go index df9c8708..cb2aa8cf 100644 --- a/pkg/credentials/store.go +++ b/pkg/credentials/store.go @@ -2,24 +2,28 @@ package credentials import ( "fmt" + "path/filepath" "regexp" + "strings" "github.com/docker/cli/cli/config/credentials" "github.com/gptscript-ai/gptscript/pkg/config" ) type Store struct { - credCtx string - cfg *config.CLIConfig + credCtx string + credHelperDirs CredentialHelperDirs + cfg *config.CLIConfig } -func NewStore(cfg *config.CLIConfig, credCtx string) (*Store, error) { +func NewStore(cfg *config.CLIConfig, credCtx, cacheDir string) (*Store, error) { if err := validateCredentialCtx(credCtx); err != nil { return nil, err } return &Store{ - credCtx: credCtx, - cfg: cfg, + credCtx: credCtx, + credHelperDirs: GetCredentialHelperDirs(cacheDir), + cfg: cfg, }, nil } @@ -103,6 +107,12 @@ func (s *Store) getStoreByHelper(helper string) (credentials.Store, error) { if helper == "" || helper == config.GPTScriptHelperPrefix+"file" { return credentials.NewFileStore(s.cfg), nil } + + // If the helper is referencing one of the credential helper programs, then reference the full path. + if strings.HasPrefix(helper, "gptscript-credential-") { + helper = filepath.Join(s.credHelperDirs.BinDir, helper) + } + return NewHelper(s.cfg, helper) } diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go new file mode 100644 index 00000000..cf3291d7 --- /dev/null +++ b/pkg/credentials/util.go @@ -0,0 +1,17 @@ +package credentials + +import ( + "path/filepath" +) + +type CredentialHelperDirs struct { + RevisionFile, BinDir, RepoDir string +} + +func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs { + return CredentialHelperDirs{ + RevisionFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "revision"), + BinDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "bin"), + RepoDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "repo"), + } +} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 6f33ef3b..a2804fa7 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/counter" "github.com/gptscript-ai/gptscript/pkg/system" "github.com/gptscript-ai/gptscript/pkg/types" @@ -19,6 +20,7 @@ type Model interface { type RuntimeManager interface { GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) + SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error } type Engine struct { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index 3d3fab2d..24498468 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -11,6 +11,7 @@ import ( "github.com/gptscript-ai/gptscript/pkg/cache" "github.com/gptscript-ai/gptscript/pkg/config" context2 "github.com/gptscript-ai/gptscript/pkg/context" + "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/llm" @@ -79,7 +80,20 @@ func New(opts *Options) (*GPTScript, error) { return nil, err } - oaiClient, err := openai.NewClient(cliCfg, opts.CredentialContext, opts.OpenAI, openai.Options{ + credStore, err := credentials.NewStore(cliCfg, opts.CredentialContext, cacheClient.CacheDir()) + if err != nil { + return nil, err + } + + if opts.Runner.RuntimeManager == nil { + opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir()) + } + + if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg, opts.Env); err != nil { + return nil, err + } + + oaiClient, err := openai.NewClient(credStore, opts.OpenAI, openai.Options{ Cache: cacheClient, SetSeed: true, }) @@ -95,11 +109,7 @@ func New(opts *Options) (*GPTScript, error) { opts.Runner.MonitorFactory = monitor.NewConsole(opts.Monitor, monitor.Options{DebugMessages: *opts.Quiet}) } - if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir()) - } - - runner, err := runner.New(registry, opts.CredentialContext, opts.Runner) + runner, err := runner.New(registry, credStore, opts.Runner) if err != nil { return nil, err } @@ -114,7 +124,7 @@ func New(opts *Options) (*GPTScript, error) { fullEnv := append(opts.Env, extraEnv...) oaiClient.SetEnvs(fullEnv) - remoteClient := remote.New(runner, fullEnv, cacheClient, cliCfg, opts.CredentialContext) + remoteClient := remote.New(runner, fullEnv, cacheClient, credStore) if err := registry.AddClient(remoteClient); err != nil { closeServer() return nil, err diff --git a/pkg/openai/client.go b/pkg/openai/client.go index 850c626b..c31e52e6 100644 --- a/pkg/openai/client.go +++ b/pkg/openai/client.go @@ -12,7 +12,6 @@ import ( openai "github.com/gptscript-ai/chat-completion-client" "github.com/gptscript-ai/gptscript/pkg/cache" - "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/counter" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/hash" @@ -44,9 +43,8 @@ type Client struct { invalidAuth bool cacheKeyBase string setSeed bool - cliCfg *config.CLIConfig - credCtx string envs []string + credStore *credentials.Store } type Options struct { @@ -88,7 +86,7 @@ func complete(opts ...Options) (result Options, err error) { return result, err } -func NewClient(cliCfg *config.CLIConfig, credCtx string, opts ...Options) (*Client, error) { +func NewClient(credStore *credentials.Store, opts ...Options) (*Client, error) { opt, err := complete(opts...) if err != nil { return nil, err @@ -96,12 +94,7 @@ func NewClient(cliCfg *config.CLIConfig, credCtx string, opts ...Options) (*Clie // If the API key is not set, try to get it from the cred store if opt.APIKey == "" && opt.BaseURL == "" { - store, err := credentials.NewStore(cliCfg, credCtx) - if err != nil { - return nil, err - } - - cred, exists, err := store.Get(BuiltinCredName) + cred, exists, err := credStore.Get(BuiltinCredName) if err != nil { return nil, err } @@ -126,8 +119,7 @@ func NewClient(cliCfg *config.CLIConfig, credCtx string, opts ...Options) (*Clie cacheKeyBase: cacheKeyBase, invalidAuth: opt.APIKey == "" && opt.BaseURL == "", setSeed: opt.SetSeed, - cliCfg: cliCfg, - credCtx: credCtx, + credStore: credStore, }, nil } @@ -548,7 +540,7 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest, } func (c *Client) RetrieveAPIKey(ctx context.Context) error { - k, err := prompt.GetModelProviderCredential(ctx, BuiltinCredName, "OPENAI_API_KEY", "Please provide your OpenAI API key:", c.credCtx, c.envs, c.cliCfg) + k, err := prompt.GetModelProviderCredential(ctx, c.credStore, BuiltinCredName, "OPENAI_API_KEY", "Please provide your OpenAI API key:", c.envs) if err != nil { return err } diff --git a/pkg/prompt/credential.go b/pkg/prompt/credential.go index f948ce10..ecafed46 100644 --- a/pkg/prompt/credential.go +++ b/pkg/prompt/credential.go @@ -4,18 +4,12 @@ import ( "context" "fmt" - "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/tidwall/gjson" ) -func GetModelProviderCredential(ctx context.Context, credName, env, message, credCtx string, envs []string, cliCfg *config.CLIConfig) (string, error) { - store, err := credentials.NewStore(cliCfg, credCtx) - if err != nil { - return "", err - } - - cred, exists, err := store.Get(credName) +func GetModelProviderCredential(ctx context.Context, credStore *credentials.Store, credName, env, message string, envs []string) (string, error) { + cred, exists, err := credStore.Get(credName) if err != nil { return "", err } @@ -30,7 +24,7 @@ func GetModelProviderCredential(ctx context.Context, credName, env, message, cre } k = gjson.Get(result, "key").String() - if err := store.Add(credentials.Credential{ + if err := credStore.Add(credentials.Credential{ ToolName: credName, Type: credentials.CredentialTypeModelProvider, Env: map[string]string{ diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index e6ec3fd2..677ad5a4 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -10,7 +10,7 @@ import ( "sync" "github.com/gptscript-ai/gptscript/pkg/cache" - "github.com/gptscript-ai/gptscript/pkg/config" + "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/engine" env2 "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/loader" @@ -27,17 +27,15 @@ type Client struct { models map[string]*openai.Client runner *runner.Runner envs []string - cliCfg *config.CLIConfig - credCtx string + credStore *credentials.Store } -func New(r *runner.Runner, envs []string, cache *cache.Client, cliCfg *config.CLIConfig, credCtx string) *Client { +func New(r *runner.Runner, envs []string, cache *cache.Client, credStore *credentials.Store) *Client { return &Client{ - cache: cache, - runner: r, - envs: envs, - cliCfg: cliCfg, - credCtx: credCtx, + cache: cache, + runner: r, + envs: envs, + credStore: credStore, } } @@ -117,7 +115,7 @@ func (c *Client) clientFromURL(ctx context.Context, apiURL string) (*openai.Clie } } - return openai.NewClient(c.cliCfg, c.credCtx, openai.Options{ + return openai.NewClient(c.credStore, openai.Options{ BaseURL: apiURL, Cache: c.cache, APIKey: key, @@ -164,7 +162,7 @@ func (c *Client) load(ctx context.Context, toolName string) (*openai.Client, err url += "/v1" } - client, err = openai.NewClient(c.cliCfg, c.credCtx, openai.Options{ + client, err = openai.NewClient(c.credStore, openai.Options{ BaseURL: url, Cache: c.cache, CacheKey: prg.EntryToolID, @@ -178,5 +176,5 @@ func (c *Client) load(ctx context.Context, toolName string) (*openai.Client, err } func (c *Client) retrieveAPIKey(ctx context.Context, env, url string) (string, error) { - return prompt.GetModelProviderCredential(ctx, url, env, fmt.Sprintf("Please provide your API key for %s", url), c.credCtx, c.envs, c.cliCfg) + return prompt.GetModelProviderCredential(ctx, c.credStore, url, env, fmt.Sprintf("Please provide your API key for %s", url), c.envs) } diff --git a/pkg/repos/get.go b/pkg/repos/get.go index da3fefdd..0d4b7bd2 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -8,12 +8,19 @@ import ( "io/fs" "os" "path/filepath" + "strings" "github.com/BurntSushi/locker" + "github.com/gptscript-ai/gptscript/pkg/config" + "github.com/gptscript-ai/gptscript/pkg/credentials" + "github.com/gptscript-ai/gptscript/pkg/loader/github" "github.com/gptscript-ai/gptscript/pkg/repos/git" + "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/types" ) +const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-helpers" + type Runtime interface { ID() string Supports(cmd []string) bool @@ -36,20 +43,79 @@ func (n noopRuntime) Setup(_ context.Context, _, _ string, _ []string) ([]string } type Manager struct { - storageDir string - gitDir string - runtimeDir string - runtimes []Runtime + storageDir string + gitDir string + runtimeDir string + credHelperDirs credentials.CredentialHelperDirs + runtimes []Runtime } func New(cacheDir string, runtimes ...Runtime) *Manager { root := filepath.Join(cacheDir, "repos") return &Manager{ - storageDir: root, - gitDir: filepath.Join(root, "git"), - runtimeDir: filepath.Join(root, "runtimes"), - runtimes: runtimes, + storageDir: root, + gitDir: filepath.Join(root, "git"), + runtimeDir: filepath.Join(root, "runtimes"), + credHelperDirs: credentials.GetCredentialHelperDirs(cacheDir), + runtimes: runtimes, + } +} + +func (m *Manager) SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error { + helperName := cliCfg.CredentialsStore + suffix := "" + if helperName == "wincred" { + suffix = ".exe" + } + + // The file helper is built-in and does not need to be compiled. + if helperName == "file" { + return nil } + + locker.Lock("gptscript-credential-helpers") + defer locker.Unlock("gptscript-credential-helpers") + + _, repo, _, err := github.Load(ctx, nil, credentialHelpersRepo) + if err != nil { + return err + } + + var needsBuild bool + + // Check the last revision shasum and see if it is different from the current one. + lastRevision, err := os.ReadFile(m.credHelperDirs.RevisionFile) + if (err == nil && strings.TrimSpace(string(lastRevision)) != repo.Revision) || errors.Is(err, fs.ErrNotExist) { + // Need to pull the latest version. + needsBuild = true + if err := git.Checkout(ctx, m.gitDir, repo.Root, repo.Revision, filepath.Join(m.credHelperDirs.RepoDir, repo.Revision)); err != nil { + return err + } + // Update the revision file to the new revision. + if err := os.WriteFile(m.credHelperDirs.RevisionFile, []byte(repo.Revision), 0644); err != nil { + return err + } + } else if err != nil { + return err + } + + if !needsBuild { + // Check for the existence of the gptscript-credential-osxkeychain binary. + // If it's there, we have no need to build it and can just return. + if _, err := os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { + return nil + } + } + + // Find the Go runtime and use it to build the credential helper. + for _, runtime := range m.runtimes { + if strings.HasPrefix(runtime.ID(), "go") { + goRuntime := runtime.(*golang.Runtime) + return goRuntime.BuildCredentialHelper(ctx, helperName, m.credHelperDirs, m.runtimeDir, repo.Revision, env) + } + } + + return fmt.Errorf("no Go runtime found to build the credential helper") } func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, env []string) (string, []string, error) { diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 0c7a48ac..e2372942 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -13,6 +13,7 @@ import ( "runtime" "strings" + "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/debugcmd" runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" @@ -51,6 +52,32 @@ func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env [] return newEnv, nil } +func (r *Runtime) BuildCredentialHelper(ctx context.Context, helperName string, credHelperDirs credentials.CredentialHelperDirs, dataRoot, revision string, env []string) error { + if helperName == "file" { + return nil + } + + suffix := "" + if helperName == "wincred" { + suffix = ".exe" + } + + binPath, err := r.getRuntime(ctx, dataRoot) + if err != nil { + return err + } + newEnv := runtimeEnv.AppendPath(env, binPath) + + log.Infof("Building credential helper %s", helperName) + cmd := debugcmd.New(ctx, filepath.Join(binPath, "go"), + "build", "-buildvcs=false", "-o", + filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix), + fmt.Sprintf("./%s/cmd/", helperName)) + cmd.Env = stripGo(append(env, newEnv...)) + cmd.Dir = filepath.Join(credHelperDirs.RepoDir, revision) + return cmd.Run() +} + func (r *Runtime) getReleaseAndDigest() (string, string, error) { scanner := bufio.NewScanner(bytes.NewReader(releasesData)) key := r.ID() + "." + runtime.GOOS + "-" + runtime.GOARCH diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 01d90bde..9ac5489f 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -11,7 +11,6 @@ import ( "time" "github.com/gptscript-ai/gptscript/pkg/builtin" - "github.com/gptscript-ai/gptscript/pkg/config" context2 "github.com/gptscript-ai/gptscript/pkg/context" "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/engine" @@ -85,22 +84,22 @@ type Runner struct { auth AuthorizerFunc factory MonitorFactory runtimeManager engine.RuntimeManager - credCtx string credMutex sync.Mutex credOverrides string + credStore *credentials.Store sequential bool } -func New(client engine.Model, credCtx string, opts ...Options) (*Runner, error) { +func New(client engine.Model, credStore *credentials.Store, opts ...Options) (*Runner, error) { opt := complete(opts...) runner := &Runner{ c: client, factory: opt.MonitorFactory, runtimeManager: opt.RuntimeManager, - credCtx: credCtx, credMutex: sync.Mutex{}, credOverrides: opt.CredentialOverride, + credStore: credStore, sequential: opt.Sequential, auth: opt.Authorizer, } @@ -787,19 +786,11 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env r.credMutex.Lock() defer r.credMutex.Unlock() - // Set up the credential store. - c, err := config.ReadCLIConfig("") - if err != nil { - return nil, fmt.Errorf("failed to read CLI config: %w", err) - } - - store, err := credentials.NewStore(c, r.credCtx) - if err != nil { - return nil, fmt.Errorf("failed to create credentials store: %w", err) - } - // Parse the credential overrides from the command line argument, if there are any. - var credOverrides map[string]map[string]string + var ( + credOverrides map[string]map[string]string + err error + ) if r.credOverrides != "" { credOverrides, err = parseCredentialOverrides(r.credOverrides) if err != nil { @@ -829,12 +820,12 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env // Only try to look up the cred if the tool is on GitHub or has an alias. // If it is a GitHub tool and has an alias, the alias overrides the tool name, so we use it as the credential name. if isGitHubTool(toolName) && credentialAlias == "" { - cred, exists, err = store.Get(toolName) + cred, exists, err = r.credStore.Get(toolName) if err != nil { return nil, fmt.Errorf("failed to get credentials for tool %s: %w", toolName, err) } } else if credentialAlias != "" { - cred, exists, err = store.Get(credentialAlias) + cred, exists, err = r.credStore.Get(credentialAlias) if err != nil { return nil, fmt.Errorf("failed to get credentials for tool %s: %w", credentialAlias, err) } @@ -900,7 +891,7 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env if (isGitHubTool(toolName) && callCtx.Program.ToolSet[credToolRefs[0].ToolID].Source.Repo != nil) || credentialAlias != "" { if isEmpty { log.Warnf("Not saving empty credential for tool %s", toolName) - } else if err := store.Add(*cred); err != nil { + } else if err := r.credStore.Add(*cred); err != nil { return nil, fmt.Errorf("failed to add credential for tool %s: %w", toolName, err) } } else { diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index cb3e58f2..013f3892 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -157,7 +157,7 @@ func NewRunner(t *testing.T) *Runner { t: t, } - run, err := runner.New(c, "default", runner.Options{ + run, err := runner.New(c, nil, runner.Options{ Sequential: true, }) require.NoError(t, err)