From 81928672e60f4987e065a07272f592ce395510b4 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:34:45 -0500 Subject: [PATCH 01/47] Add separate require line for root module. Separating it to avoid potential merge conflicts when surrounding lines change as we expect CI to update its psuedo version. --- core/scripts/go.mod | 5 ++++- deployment/go.mod | 5 ++++- integration-tests/go.mod | 5 ++++- integration-tests/load/go.mod | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 028eb9fac9e..5679981056a 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -5,6 +5,10 @@ go 1.22.8 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ +// Using a separate inline `require` here to avoid surrounding line changes +// creating potential merge conflicts. +require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 + replace github.com/smartcontractkit/chainlink/deployment => ../../deployment require ( @@ -26,7 +30,6 @@ require ( github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-common v0.3.1-0.20241114134822-aadff98ef068 github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 - github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 diff --git a/deployment/go.mod b/deployment/go.mod index 73e23aa83f3..31a5cb89c99 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -5,6 +5,10 @@ go 1.22.8 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../ +// Using a separate inline `require` here to avoid surrounding line changes +// creating potential merge conflicts. +require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 + require ( github.com/Khan/genqlient v0.7.0 github.com/Masterminds/semver/v3 v3.3.0 @@ -26,7 +30,6 @@ require ( github.com/smartcontractkit/chainlink-common v0.3.1-0.20241114134822-aadff98ef068 github.com/smartcontractkit/chainlink-protos/job-distributor v0.4.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 - github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/stretchr/testify v1.9.0 github.com/test-go/testify v1.1.4 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index b5948028f32..92644361bea 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -7,6 +7,10 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ replace github.com/smartcontractkit/chainlink/deployment => ../deployment +// Using a separate inline `require` here to avoid surrounding line changes +// creating potential merge conflicts. +require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 + require ( dario.cat/mergo v1.0.1 github.com/AlekSi/pointer v1.1.0 @@ -45,7 +49,6 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 - github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 7398abc5af9..1cb20794061 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -9,6 +9,10 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment replace github.com/smartcontractkit/chainlink/integration-tests => ../ +// Using a separate inline `require` here to avoid surrounding line changes +// creating potential merge conflicts. +require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 + require ( github.com/K-Phoen/grabana v0.22.2 github.com/ethereum/go-ethereum v1.14.11 @@ -23,7 +27,6 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241030133659-9ec788e78b4f - github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20241009055228-33d0c0bf38de github.com/stretchr/testify v1.9.0 github.com/wiremock/go-wiremock v1.9.0 From 8f5b382be6d82f756ec43f33880eab93a39c8618 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:45:06 -0500 Subject: [PATCH 02/47] Create tool to update certain module deps to pin to latest trunk commit --- tools/gomod-required-updater/README.md | 31 +++ .../cmd/gomod-required-updater/main.go | 49 +++++ tools/gomod-required-updater/go.mod | 9 + tools/gomod-required-updater/go.sum | 4 + .../internal/updater/config.go | 98 +++++++++ .../internal/updater/config_test.go | 106 ++++++++++ .../internal/updater/git_operator.go | 68 ++++++ .../internal/updater/interfaces.go | 33 +++ .../internal/updater/system_operator.go | 65 ++++++ .../internal/updater/updater.go | 138 +++++++++++++ .../internal/updater/updater_test.go | 194 ++++++++++++++++++ 11 files changed, 795 insertions(+) create mode 100644 tools/gomod-required-updater/README.md create mode 100644 tools/gomod-required-updater/cmd/gomod-required-updater/main.go create mode 100644 tools/gomod-required-updater/go.mod create mode 100644 tools/gomod-required-updater/go.sum create mode 100644 tools/gomod-required-updater/internal/updater/config.go create mode 100644 tools/gomod-required-updater/internal/updater/config_test.go create mode 100644 tools/gomod-required-updater/internal/updater/git_operator.go create mode 100644 tools/gomod-required-updater/internal/updater/interfaces.go create mode 100644 tools/gomod-required-updater/internal/updater/system_operator.go create mode 100644 tools/gomod-required-updater/internal/updater/updater.go create mode 100644 tools/gomod-required-updater/internal/updater/updater_test.go diff --git a/tools/gomod-required-updater/README.md b/tools/gomod-required-updater/README.md new file mode 100644 index 00000000000..082dd99acce --- /dev/null +++ b/tools/gomod-required-updater/README.md @@ -0,0 +1,31 @@ +# gomod-required-updater + +## Installation + +The installed binary will be placed in your `$GOPATH/bin` directory. Make sure this directory is in your system's PATH to run the command from anywhere. + +```shell +go install github.com/smartcontractkit/chainlink/tools/gomod-required-updater +``` + +## Config + +```toml +# Add any modules in here that you want to update the required version for. Can optionally be provided via the command line flag `-module`. +modules = [ + "github.com/smartcontractkit/chainlink/v2" +] +``` + +## Usage + +```shell +# Using config file +gomod-required-updater -config modules.toml +# Using command line modules +gomod-required-updater -module github.com/org/repo1 -module github.com/org/repo2 +# Mixed (command line takes precedence) +gomod-required-updater -config modules.toml -module github.com/org/repo1 +``` + +Even though both `-config` and `-module` are provided, only `github.com/org/repo1` will be used because the command line `-module` flag takes precedence, and the config file is completely ignored when any `-module` flags are present. diff --git a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go new file mode 100644 index 00000000000..90cb7ffa506 --- /dev/null +++ b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github.com/smartcontractkit/chainlink/tools/gomod-required-updater/internal/updater" +) + +var version = "dev" +var usage = `gomod-required-updater version %s + +Usage: + gomod-required-updater [flags] + +Examples: + # Update modules specified in config file + gomod-required-updater -config modules.toml + + # Update specific modules + gomod-required-updater -module github.com/org/repo1 -module github.com/org/repo2 + + # Dry run with specific modules + gomod-required-updater -dry-run -module github.com/org/repo1 +` + +func main() { + cfg, err := updater.ParseFlags(os.Args[1:], version) + if err != nil { + fmt.Fprintf(os.Stderr, usage, version) + log.Fatal(err) + } + + if cfg.ShowVersion { + fmt.Printf("gomod-required-updater version %s\n", version) + os.Exit(0) + } + + u := updater.New( + updater.NewGitOperator(), + updater.NewSystemOperator(), + cfg, + ) + + if err := u.Run(); err != nil { + log.Fatal(err) + } +} \ No newline at end of file diff --git a/tools/gomod-required-updater/go.mod b/tools/gomod-required-updater/go.mod new file mode 100644 index 00000000000..b6caaedfb33 --- /dev/null +++ b/tools/gomod-required-updater/go.mod @@ -0,0 +1,9 @@ +module github.com/smartcontractkit/chainlink/tools/gomod-required-updater + +go 1.23 + +require ( + github.com/BurntSushi/toml v1.4.0 + golang.org/x/mod v0.22.0 + +) diff --git a/tools/gomod-required-updater/go.sum b/tools/gomod-required-updater/go.sum new file mode 100644 index 00000000000..c568698e71a --- /dev/null +++ b/tools/gomod-required-updater/go.sum @@ -0,0 +1,4 @@ +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go new file mode 100644 index 00000000000..1adaf8aef92 --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -0,0 +1,98 @@ +package updater + +import ( + "flag" + "fmt" + "strings" + + "github.com/BurntSushi/toml" +) + +type TOMLConfig struct { + Modules []string `toml:"modules"` +} + +type Config struct { + ModulesToUpdate []string + RepoRemote string + BranchTrunk string + DryRun bool + ConfigFile string + RootPath string + ShowVersion bool + modulesSource string +} + +func (c *Config) Validate() error { + if !c.ShowVersion { + if len(c.ModulesToUpdate) == 0 { + return fmt.Errorf("%w: no modules specified to update (use -module flag or config file)", ErrInvalidConfig) + } + if c.RepoRemote == "" { + return fmt.Errorf("%w: repo remote cannot be empty", ErrInvalidConfig) + } + if c.BranchTrunk == "" { + return fmt.Errorf("%w: branch trunk cannot be empty", ErrInvalidConfig) + } + } + return nil +} + +func ParseFlags(args []string, version string) (*Config, error) { + flags := flag.NewFlagSet("gomod-required-updater", flag.ContinueOnError) + + cfg := &Config{} + var cliModules arrayFlags // Define custom flag type for multiple -module flags + + flags.StringVar(&cfg.RepoRemote, "repo-remote", "origin", "The name of the repo remote") + flags.StringVar(&cfg.BranchTrunk, "branch-trunk", "develop", "The name of the trunk branch") + flags.BoolVar(&cfg.DryRun, "dry-run", false, "Print what would be done without making changes") + flags.StringVar(&cfg.ConfigFile, "config", "", "Path to TOML config file (optional if using -module)") + flags.StringVar(&cfg.RootPath, "root", ".", "Root path to start scanning for go.mod files") + flags.BoolVar(&cfg.ShowVersion, "version", false, "Show version information") + flags.Var(&cliModules, "module", "Module to update (can be specified multiple times)") + + if err := flags.Parse(args); err != nil { + return nil, err + } + + // CLI modules take precedence + if len(cliModules) > 0 { + cfg.ModulesToUpdate = cliModules + cfg.modulesSource = "command line" + } else if cfg.ConfigFile != "" { + // Only load config file if no CLI modules specified + modules, err := loadTOMLConfig(cfg.ConfigFile) + if err != nil { + return nil, fmt.Errorf("failed to load config: %w", err) + } + cfg.ModulesToUpdate = modules + cfg.modulesSource = "config file" + } + + if err := cfg.Validate(); err != nil { + return nil, err + } + + return cfg, nil +} + +// arrayFlags allows for repeated flag values +type arrayFlags []string + +func (i *arrayFlags) String() string { + return strings.Join(*i, ", ") +} + +func (i *arrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +} + +func loadTOMLConfig(path string) ([]string, error) { + var cfg TOMLConfig + if _, err := toml.DecodeFile(path, &cfg); err != nil { + return nil, fmt.Errorf("failed to decode TOML: %w", err) + } + return cfg.Modules, nil +} \ No newline at end of file diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-required-updater/internal/updater/config_test.go new file mode 100644 index 00000000000..c093b5a3cc2 --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/config_test.go @@ -0,0 +1,106 @@ +package updater + +import "testing" + +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + cfg *Config + wantErr bool + }{ + { + name: "valid config", + cfg: &Config{ + ModulesToUpdate: []string{"test.com/mod"}, + RepoRemote: "origin", + BranchTrunk: "main", + }, + wantErr: false, + }, + { + name: "missing modules", + cfg: &Config{ + RepoRemote: "origin", + BranchTrunk: "main", + }, + wantErr: true, + }, + { + name: "version flag bypasses validation", + cfg: &Config{ + ShowVersion: true, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestParseFlags(t *testing.T) { + tests := []struct { + name string + args []string + wantCfg *Config + wantErr bool + }{ + { + name: "multiple modules via CLI", + args: []string{"-module", "mod1.com", "-module", "mod2.com"}, + wantCfg: &Config{ + ModulesToUpdate: []string{"mod1.com", "mod2.com"}, + RepoRemote: "origin", + BranchTrunk: "develop", + }, + wantErr: false, + }, + { + name: "show version", + args: []string{"-version"}, + wantCfg: &Config{ + ShowVersion: true, + RepoRemote: "origin", // Default value + BranchTrunk: "develop", // Default value + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseFlags(tt.args, "test-version") + if (err != nil) != tt.wantErr { + t.Errorf("ParseFlags() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + + if got.RepoRemote != tt.wantCfg.RepoRemote { + t.Errorf("ParseFlags() RepoRemote = %v, want %v", got.RepoRemote, tt.wantCfg.RepoRemote) + } + if got.BranchTrunk != tt.wantCfg.BranchTrunk { + t.Errorf("ParseFlags() BranchTrunk = %v, want %v", got.BranchTrunk, tt.wantCfg.BranchTrunk) + } + if got.ShowVersion != tt.wantCfg.ShowVersion { + t.Errorf("ParseFlags() ShowVersion = %v, want %v", got.ShowVersion, tt.wantCfg.ShowVersion) + } + if len(got.ModulesToUpdate) != len(tt.wantCfg.ModulesToUpdate) { + t.Errorf("ParseFlags() ModulesToUpdate length = %v, want %v", len(got.ModulesToUpdate), len(tt.wantCfg.ModulesToUpdate)) + } + for i, module := range got.ModulesToUpdate { + if module != tt.wantCfg.ModulesToUpdate[i] { + t.Errorf("ParseFlags() ModulesToUpdate[%d] = %v, want %v", i, module, tt.wantCfg.ModulesToUpdate[i]) + } + } + }) + } +} diff --git a/tools/gomod-required-updater/internal/updater/git_operator.go b/tools/gomod-required-updater/internal/updater/git_operator.go new file mode 100644 index 00000000000..584850cdbd7 --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/git_operator.go @@ -0,0 +1,68 @@ +package updater + +import ( + "fmt" + "log" + "os/exec" + "strings" + "time" +) + +type gitOperator struct{} + +// NewGitOperator creates a new instance of GitOperator +func NewGitOperator() GitOperator { + return &gitOperator{} +} + +// GetSHA returns the SHA of a given branch in a remote repository +func (g *gitOperator) GetSHA(remote, branch string) (string, error) { + // Use the full ref path to get exact match + cmd := exec.Command("git", "ls-remote", remote, "refs/heads/"+branch) + log.Printf("Running git command: git ls-remote %s refs/heads/%s", remote, branch) + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("%w: failed to get remote SHA: %v", ErrGitOperation, err) + } + + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(lines) == 0 || lines[0] == "" { + return "", fmt.Errorf("%w: no SHA found for branch %s in remote %s", ErrGitOperation, branch, remote) + } + + // Parse the first line that exactly matches our ref + for _, line := range lines { + parts := strings.Split(line, "\t") + if len(parts) != 2 { + continue + } + sha := strings.TrimSpace(parts[0]) + ref := strings.TrimSpace(parts[1]) + + if ref == "refs/heads/"+branch { + log.Printf("Found remote SHA: %s for ref: %s", sha, ref) + return sha, nil + } + } + + return "", fmt.Errorf("%w: no exact match found for refs/heads/%s", ErrGitOperation, branch) +} + +// GetCommitDate returns the commit date of a given SHA used for in the go.mod +// psuedo-version for the module. +func (g *gitOperator) GetCommitDate(sha string) (time.Time, error) { + // Get commit date in ISO 8601 time format + cmd := exec.Command("git", "show", "-s", "--format=%cI", sha) + output, err := cmd.Output() + if err != nil { + return time.Time{}, fmt.Errorf("failed to get commit date: %w", err) + } + + dateStr := strings.TrimSpace(string(output)) + t, err := time.Parse(time.RFC3339, dateStr) + if err != nil { + return time.Time{}, fmt.Errorf("failed to parse date: %w", err) + } + + return t, nil +} diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go new file mode 100644 index 00000000000..25b5ecd684e --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -0,0 +1,33 @@ +package updater + +import ( + "fmt" + "time" +) + +// Errors +var ( + ErrInvalidConfig = fmt.Errorf("invalid configuration") + ErrGitOperation = fmt.Errorf("git operation failed") + ErrFileOperation = fmt.Errorf("file operation failed") +) + +type GitOperator interface { + GetSHA(remote, branch string) (string, error) + GetCommitDate(sha string) (time.Time, error) +} + +type SystemOperator interface { + ReadFile(path string) ([]byte, error) + WriteFile(filename string, data []byte, perm uint32) error + Walk(root string, fn func(path string, isDir bool) error) error + Chdir(dir string) error + Getwd() (string, error) + RunCommand(name string, args ...string) error +} + +type Updater struct { + git GitOperator + system SystemOperator + config *Config +} \ No newline at end of file diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go new file mode 100644 index 00000000000..302a0466c2c --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -0,0 +1,65 @@ +package updater + +import ( + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" +) + +type systemOperator struct { + stdout io.Writer + stderr io.Writer +} + +func NewSystemOperator() SystemOperator { + return &systemOperator{ + stdout: os.Stdout, + stderr: os.Stderr, + } +} + +// For testing +func NewSystemOperatorWithIO(stdout, stderr io.Writer) SystemOperator { + return &systemOperator{ + stdout: stdout, + stderr: stderr, + } +} + +func (so *systemOperator) ReadFile(path string) ([]byte, error) { + return os.ReadFile(path) +} + +func (so *systemOperator) WriteFile(filename string, data []byte, perm uint32) error { + return os.WriteFile(filename, data, os.FileMode(perm)) +} + +func (so *systemOperator) Walk(root string, fn func(path string, isDir bool) error) error { + return filepath.Walk(root, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + return fn(path, info.IsDir()) + }) +} + +func (so *systemOperator) Chdir(dir string) error { + return os.Chdir(dir) +} + +func (so *systemOperator) Getwd() (string, error) { + return os.Getwd() +} + +func (so *systemOperator) RunCommand(name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdout = so.stdout + cmd.Stderr = so.stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%w: %v", ErrFileOperation, err) + } + return nil +} \ No newline at end of file diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go new file mode 100644 index 00000000000..55414f6b52e --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -0,0 +1,138 @@ +package updater + +import ( + "fmt" + "log" + "path/filepath" + "regexp" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +func New(git GitOperator, system SystemOperator, config *Config) *Updater { + return &Updater{ + git: git, + system: system, + config: config, + } +} + +func (u *Updater) Run() error { + log.Printf("Starting update process with remote '%s' and branch '%s'", u.config.RepoRemote, u.config.BranchTrunk) + + absRoot, err := filepath.Abs(u.config.RootPath) + if err != nil { + return fmt.Errorf("failed to resolve root path: %w", err) + } + + log.Printf("Fetching latest SHA from %s/%s", u.config.RepoRemote, u.config.BranchTrunk) + sha, err := u.git.GetSHA(u.config.RepoRemote, u.config.BranchTrunk) + if err != nil { + return fmt.Errorf("failed to get SHA: %w", err) + } + log.Printf("Using SHA: %s", sha) + + return u.system.Walk(absRoot, func(path string, isDir bool) error { + if filepath.Base(path) == "go.mod" { + for _, module := range u.config.ModulesToUpdate { + if err := u.updateGoMod(path, module, sha); err != nil { + return fmt.Errorf("error updating %s: %w", path, err) + } + } + } + return nil + }) +} + +func (u *Updater) updateGoMod(path, modulePath, sha string) error { + log.Printf("Processing %s for module %s", path, modulePath) + + content, err := u.system.ReadFile(path) + if (err != nil) { + return err + } + + f, err := modfile.Parse(path, content, nil) + if err != nil { + return err + } + + moduleExists := false + var currentVersion string + for _, req := range f.Require { + if req.Mod.Path == modulePath { + moduleExists = true + currentVersion = req.Mod.Version + break + } + } + + if !moduleExists { + log.Printf("Skipping %s: module %s not found", path, modulePath) + return nil + } + + log.Printf("Current version: %s", currentVersion) + + commitDate, err := u.git.GetCommitDate(sha) + if err != nil { + return fmt.Errorf("failed to get commit date: %w", err) + } + + shortSHA := sha[:12] + versionPrefix := parseModuleVersion(modulePath) + pseudoVersion := module.PseudoVersion("v"+versionPrefix, "", commitDate, shortSHA) + + log.Printf("Updating to version: %s", pseudoVersion) + + if u.config.DryRun { + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, currentVersion, pseudoVersion) + return nil + } + + if err := f.AddRequire(modulePath, pseudoVersion); err != nil { + return fmt.Errorf("failed to add requirement: %w", err) + } + + newContent, err := f.Format() + if err != nil { + return fmt.Errorf("failed to format go.mod: %w", err) + } + + if err := u.system.WriteFile(path, newContent, 0644); err != nil { + return fmt.Errorf("failed to write go.mod: %w", err) + } + + dir := filepath.Dir(path) + origDir, err := u.system.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + + if err := u.system.Chdir(dir); err != nil { + return fmt.Errorf("failed to change to directory %s: %w", dir, err) + } + // Use defer to ensure we always return to the initial directory. + defer func() { + if err := u.system.Chdir(origDir); err != nil { + log.Printf("Warning: failed to return to original directory: %v", err) + } + }() + + log.Printf("Running go mod tidy in %s", dir) + if err := u.system.RunCommand("go", "mod", "tidy"); err != nil { + return fmt.Errorf("go mod tidy failed: %w", err) + } + + return nil +} + +func parseModuleVersion(modulePath string) string { + ver := module.Version{Path: modulePath} + re := regexp.MustCompile(`/v(\d+)$`) + if match := re.FindStringSubmatch(ver.Path); match != nil { + return match[1] + } + return "0" +} \ No newline at end of file diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go new file mode 100644 index 00000000000..b9d4fd9758e --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -0,0 +1,194 @@ +package updater + +import ( + "fmt" + "testing" + "time" +) + +// Mock implementations +type mockGitOperator struct { + sha string + commitDate time.Time + err error +} + +func (m *mockGitOperator) GetSHA(remote, branch string) (string, error) { + if m.err != nil { + return "", m.err + } + return m.sha, nil +} + +func (m *mockGitOperator) GetCommitDate(sha string) (time.Time, error) { + if m.err != nil { + return time.Time{}, m.err + } + return m.commitDate, nil +} + +type mockSystemOperator struct { + files map[string][]byte + walkFn func(path string, isDir bool) error + commands []string + err error +} + +func newMockSystemOperator() *mockSystemOperator { + return &mockSystemOperator{ + files: make(map[string][]byte), + } +} + +func (m *mockSystemOperator) ReadFile(path string) ([]byte, error) { + if m.err != nil { + return nil, m.err + } + content, ok := m.files[path] + if !ok { + return nil, fmt.Errorf("file not found: %s", path) + } + return content, nil +} + +func (m *mockSystemOperator) WriteFile(path string, data []byte, perm uint32) error { + if m.err != nil { + return m.err + } + m.files[path] = data + return nil +} + +func (m *mockSystemOperator) Walk(root string, fn func(path string, isDir bool) error) error { + if m.walkFn != nil { + return m.walkFn(root, true) + } + return nil +} + +func (m *mockSystemOperator) Chdir(dir string) error { return m.err } +func (m *mockSystemOperator) Getwd() (string, error) { return "/mock/dir", m.err } +func (m *mockSystemOperator) RunCommand(name string, args ...string) error { + if m.err != nil { + return m.err + } + m.commands = append(m.commands, name+" "+fmt.Sprint(args)) + return nil +} + +func TestUpdater_Run(t *testing.T) { + tests := []struct { + name string + config *Config + gitOp *mockGitOperator + sysOp *mockSystemOperator + wantErr bool + wantFile string // Expected file content after update + }{ + { + name: "successful update", + config: &Config{ + ModulesToUpdate: []string{"github.com/example/module"}, + RepoRemote: "origin", + BranchTrunk: "main", + RootPath: ".", + }, + gitOp: &mockGitOperator{ + sha: "abc123def456", + commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`module test +require github.com/example/module v0.0.0-20230101000000-123456789abc +`) + m.walkFn = func(path string, isDir bool) error { + return nil + } + return m + }(), + wantErr: false, + }, + { + name: "handles multiple go.mod files", + config: &Config{ + ModulesToUpdate: []string{"test.com/mod"}, + RepoRemote: "origin", + BranchTrunk: "main", + }, + gitOp: &mockGitOperator{ + sha: "def456", + commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["dir1/go.mod"] = []byte(`require test.com/mod v1.0.0`) + m.files["dir2/go.mod"] = []byte(`require test.com/mod v1.0.0`) + paths := []string{"dir1/go.mod", "dir2/go.mod"} + i := 0 + m.walkFn = func(path string, isDir bool) error { + if i < len(paths) { + return nil + } + return nil + } + return m + }(), + wantErr: false, + }, + { + name: "handles module with version suffix", + config: &Config{ + ModulesToUpdate: []string{"test.com/mod/v2"}, + RepoRemote: "origin", + BranchTrunk: "main", + }, + gitOp: &mockGitOperator{ + sha: "abc123", + commitDate: time.Now(), + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`require test.com/mod/v2 v2.0.0`) + return m + }(), + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := New(tt.gitOp, tt.sysOp, tt.config) + err := u.Run() + if (err != nil) != tt.wantErr { + t.Errorf("Updater.Run() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestUpdater_UpdateMultipleModules(t *testing.T) { + sysOp := newMockSystemOperator() + sysOp.files["go.mod"] = []byte(` +module test +require ( + mod1.com v1.0.0 + mod2.com v1.0.0 +)`) + + gitOp := &mockGitOperator{ + sha: "abc123", + commitDate: time.Now(), + } + + cfg := &Config{ + ModulesToUpdate: []string{"mod1.com", "mod2.com"}, + RepoRemote: "origin", + BranchTrunk: "main", + } + + u := New(gitOp, sysOp, cfg) + if err := u.Run(); err != nil { + t.Errorf("unexpected error: %v", err) + } +} \ No newline at end of file From 81968a861fcf949f62e771d0b3a925e94e61a02e Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:45:30 -0500 Subject: [PATCH 03/47] Create new makefile target to use new tool to update module deps to pin --- GNUmakefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/GNUmakefile b/GNUmakefile index 592183923e2..3c80f02cbb3 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -30,6 +30,10 @@ gomod: ## Ensure chainlink's go dependencies are installed. gomodtidy: gomods ## Run go mod tidy on all modules. gomods tidy +.PHONY: gomodrequiredupdater +gomodrequiredupdater: ## Update go.mod files containing certain required dependencies to use latest psuedo-versions from trunk. + cd tools/gomod-required-updater && go run cmd/gomod-required-updater/main.go -module github.com/smartcontractkit/chainlink/v2 -root ../.. + .PHONY: docs docs: ## Install and run pkgsite to view Go docs go install golang.org/x/pkgsite/cmd/pkgsite@latest From 1a1d9d379d3cc1054d62eb0eb9e38dbd55bcaee1 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 19 Nov 2024 07:49:10 -0500 Subject: [PATCH 04/47] Add support for additional local module replacements --- core/scripts/go.mod | 12 +++++++----- deployment/go.mod | 2 +- integration-tests/go.mod | 8 +++++--- integration-tests/load/go.mod | 10 ++++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 5679981056a..3814af1b709 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -5,12 +5,15 @@ go 1.22.8 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ -// Using a separate inline `require` here to avoid surrounding line changes -// creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 - replace github.com/smartcontractkit/chainlink/deployment => ../../deployment +// Using a separate `require` here to avoid surrounding line changes +// creating potential merge conflicts. +require ( + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d +) + require ( github.com/docker/docker v27.3.1+incompatible github.com/docker/go-connections v0.5.0 @@ -29,7 +32,6 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chainlink-automation v0.8.1 github.com/smartcontractkit/chainlink-common v0.3.1-0.20241114134822-aadff98ef068 - github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 diff --git a/deployment/go.mod b/deployment/go.mod index 31a5cb89c99..34e8ec80961 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -7,7 +7,7 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ // Using a separate inline `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d require ( github.com/Khan/genqlient v0.7.0 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 92644361bea..bb5469aa3de 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -7,9 +7,12 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ replace github.com/smartcontractkit/chainlink/deployment => ../deployment -// Using a separate inline `require` here to avoid surrounding line changes +// Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 +require ( + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d +) require ( dario.cat/mergo v1.0.1 @@ -48,7 +51,6 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 - github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 1cb20794061..6424b05a4c0 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -9,9 +9,13 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment replace github.com/smartcontractkit/chainlink/integration-tests => ../ -// Using a separate inline `require` here to avoid surrounding line changes +// Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 +require ( + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d +) require ( github.com/K-Phoen/grabana v0.22.2 @@ -25,8 +29,6 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.13 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2 - github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000 - github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241030133659-9ec788e78b4f github.com/smartcontractkit/tdh2/go/tdh2 v0.0.0-20241009055228-33d0c0bf38de github.com/stretchr/testify v1.9.0 github.com/wiremock/go-wiremock v1.9.0 From 1e3811ce9d84e6a962f1398ec6101c0413cecf87 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:04:32 -0500 Subject: [PATCH 05/47] Add support for automatically updating locally replaced modules --- tools/gomod-required-updater/README.md | 79 ++++++++-- .../internal/updater/config.go | 17 ++- .../internal/updater/config_test.go | 9 ++ .../internal/updater/git_operator.go | 21 +++ .../internal/updater/interfaces.go | 1 + .../internal/updater/updater.go | 87 +++++++++++ .../internal/updater/updater_test.go | 142 ++++++++++++++++-- 7 files changed, 327 insertions(+), 29 deletions(-) diff --git a/tools/gomod-required-updater/README.md b/tools/gomod-required-updater/README.md index 082dd99acce..dbd89ef5cc0 100644 --- a/tools/gomod-required-updater/README.md +++ b/tools/gomod-required-updater/README.md @@ -1,31 +1,82 @@ # gomod-required-updater -## Installation +Updates required module versions in go.mod files to match the latest git SHA from a remote branch. -The installed binary will be placed in your `$GOPATH/bin` directory. Make sure this directory is in your system's PATH to run the command from anywhere. +## Features -```shell -go install github.com/smartcontractkit/chainlink/tools/gomod-required-updater -``` +- Update modules to latest SHA from specified branch +- Auto-detect and update modules with local replace directives +- Update multiple modules at once +- Preview changes with dry run mode +- Configure via TOML or command line flags + +## Configuration -## Config +Optional TOML config. ```toml -# Add any modules in here that you want to update the required version for. Can optionally be provided via the command line flag `-module`. +# List of modules to update modules = [ "github.com/smartcontractkit/chainlink/v2" ] ``` -## Usage +Command Line Flags: ```shell -# Using config file -gomod-required-updater -config modules.toml -# Using command line modules +Required (one of): + -module Module to update (can be specified multiple times) + -config Path to TOML config file + -update-org-modules Auto-detect and update modules with local replaces + +Optional: + -repo-remote Git remote to use (default: origin) + -branch-trunk Branch to get SHA from (default: develop) + -root Root path for searching go.mod files + -dry-run Preview changes without applying them +``` + +## Installation + +The installed binary will be placed in your `$GOPATH/bin` directory. Make sure this directory is in your system's PATH to run the command from anywhere. + +```shell +go install github.com/smartcontractkit/chainlink/tools/gomod-required-updater +``` + +## Usage Examples + +Update Specific Modules: + +```shell +# Update single module +gomod-required-updater -module github.com/org/repo` + +# Update multiple modules gomod-required-updater -module github.com/org/repo1 -module github.com/org/repo2 -# Mixed (command line takes precedence) -gomod-required-updater -config modules.toml -module github.com/org/repo1 + +# Update using config file +gomod-required-updater -config modules.toml + +# Using different remote/branch +gomod-required-updater -module github.com/org/repo -repo-remote upstream -branch-trunk main ``` -Even though both `-config` and `-module` are provided, only `github.com/org/repo1` will be used because the command line `-module` flag takes precedence, and the config file is completely ignored when any `-module` flags are present. +Auto-detect and Update Local Modules: + +```shell +# Update all local modules that have replace directives +gomod-required-updater -update-org-modules + +# Preview changes first +gomod-required-updater -update-org-modules -dry-run +``` + +## Notes + +- When using multiple module sources, precedence is: + 1. Command line `-module` flags + 2. Config file modules + 3. Auto-detected modules via `-update-org-modules` flag +- Use the `-dry-run` flag to safely preview changes +- Local replace directives are preserved during updates diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go index 1adaf8aef92..cc4f3c09e51 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -21,11 +21,15 @@ type Config struct { RootPath string ShowVersion bool modulesSource string + UpdateOrgModules bool // Update modules from same org/repo with local replaces + OrgName string // GitHub organization name + RepoName string // Repository name } func (c *Config) Validate() error { if !c.ShowVersion { - if len(c.ModulesToUpdate) == 0 { + // Skip module validation if using UpdateOrgModules + if !c.UpdateOrgModules && len(c.ModulesToUpdate) == 0 { return fmt.Errorf("%w: no modules specified to update (use -module flag or config file)", ErrInvalidConfig) } if c.RepoRemote == "" { @@ -51,6 +55,7 @@ func ParseFlags(args []string, version string) (*Config, error) { flags.StringVar(&cfg.RootPath, "root", ".", "Root path to start scanning for go.mod files") flags.BoolVar(&cfg.ShowVersion, "version", false, "Show version information") flags.Var(&cliModules, "module", "Module to update (can be specified multiple times)") + flags.BoolVar(&cfg.UpdateOrgModules, "update-org-modules", false, "Update modules from same org/repo that have local replace directives") if err := flags.Parse(args); err != nil { return nil, err @@ -70,6 +75,16 @@ func ParseFlags(args []string, version string) (*Config, error) { cfg.modulesSource = "config file" } + if cfg.UpdateOrgModules { + gitOp := NewGitOperator() + org, repo, err := gitOp.GetRepoInfo(cfg.RepoRemote) + if err != nil { + return nil, fmt.Errorf("failed to get repo info: %w", err) + } + cfg.OrgName = org + cfg.RepoName = repo + } + if err := cfg.Validate(); err != nil { return nil, err } diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-required-updater/internal/updater/config_test.go index c093b5a3cc2..77a696d18a7 100644 --- a/tools/gomod-required-updater/internal/updater/config_test.go +++ b/tools/gomod-required-updater/internal/updater/config_test.go @@ -32,6 +32,15 @@ func TestConfig_Validate(t *testing.T) { }, wantErr: false, }, + { + name: "update-org-modules bypasses module validation", + cfg: &Config{ + UpdateOrgModules: true, + RepoRemote: "origin", + BranchTrunk: "main", + }, + wantErr: false, + }, } for _, tt := range tests { diff --git a/tools/gomod-required-updater/internal/updater/git_operator.go b/tools/gomod-required-updater/internal/updater/git_operator.go index 584850cdbd7..e5eb3c31b87 100644 --- a/tools/gomod-required-updater/internal/updater/git_operator.go +++ b/tools/gomod-required-updater/internal/updater/git_operator.go @@ -66,3 +66,24 @@ func (g *gitOperator) GetCommitDate(sha string) (time.Time, error) { return t, nil } + +func (g *gitOperator) GetRepoInfo(remote string) (org, repo string, err error) { + cmd := exec.Command("git", "config", "--get", fmt.Sprintf("remote.%s.url", remote)) + output, err := cmd.Output() + if err != nil { + return "", "", fmt.Errorf("failed to get repo info for remote %s: %w", remote, err) + } + + // Handle different URL formats: + // https://github.com/org/repo.git + // git@github.com:org/repo.git + url := strings.TrimSpace(string(output)) + parts := strings.Split(strings.TrimSuffix(url, ".git"), "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("unexpected git URL format from remote %s: %s", remote, url) + } + + repo = parts[len(parts)-1] + org = parts[len(parts)-2] + return org, repo, nil +} diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index 25b5ecd684e..a4fd5fc17c8 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -15,6 +15,7 @@ var ( type GitOperator interface { GetSHA(remote, branch string) (string, error) GetCommitDate(sha string) (time.Time, error) + GetRepoInfo(remote string) (org, repo string, err error) } type SystemOperator interface { diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 55414f6b52e..379d82ac9d6 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -5,6 +5,7 @@ import ( "log" "path/filepath" "regexp" + "strings" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -26,6 +27,25 @@ func (u *Updater) Run() error { return fmt.Errorf("failed to resolve root path: %w", err) } + // If org modules update is enabled, collect all modules with local replaces + if u.config.UpdateOrgModules { + modulesToAdd, err := u.findLocalReplaceModules(absRoot) + if err != nil { + return fmt.Errorf("failed to find local replace modules: %w", err) + } + if len(modulesToAdd) > 0 { + log.Printf("Found %d modules with local replace directives", len(modulesToAdd)) + u.config.ModulesToUpdate = append(u.config.ModulesToUpdate, modulesToAdd...) + } + + rootMod, err := u.findRootModule() + if err != nil { + return fmt.Errorf("failed to find root module: %w", err) + } + u.config.ModulesToUpdate = append(u.config.ModulesToUpdate, rootMod) + } + + // Get SHA after collecting modules log.Printf("Fetching latest SHA from %s/%s", u.config.RepoRemote, u.config.BranchTrunk) sha, err := u.git.GetSHA(u.config.RepoRemote, u.config.BranchTrunk) if err != nil { @@ -45,6 +65,20 @@ func (u *Updater) Run() error { }) } +func (u *Updater) findRootModule() (string, error) { + content, err := u.system.ReadFile("go.mod") + if err != nil { + return "", fmt.Errorf("failed to read root go.mod: %w", err) + } + + f, err := modfile.Parse("go.mod", content, nil) + if err != nil { + return "", fmt.Errorf("failed to parse root go.mod: %w", err) + } + + return f.Module.Mod.Path, nil +} + func (u *Updater) updateGoMod(path, modulePath, sha string) error { log.Printf("Processing %s for module %s", path, modulePath) @@ -68,6 +102,17 @@ func (u *Updater) updateGoMod(path, modulePath, sha string) error { } } + // Check for replace directive if updating org modules + if u.config.UpdateOrgModules { + for _, rep := range f.Replace { + if rep.Old.Path == modulePath && isLocalPath(rep.New.Path) { + log.Printf("Found local replace for %s => %s", modulePath, rep.New.Path) + moduleExists = true + break + } + } + } + if !moduleExists { log.Printf("Skipping %s: module %s not found", path, modulePath) return nil @@ -128,6 +173,42 @@ func (u *Updater) updateGoMod(path, modulePath, sha string) error { return nil } +func (u *Updater) findLocalReplaceModules(root string) ([]string, error) { + var modules []string + seen := make(map[string]bool) + orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) + + err := u.system.Walk(root, func(path string, isDir bool) error { + if filepath.Base(path) != "go.mod" { + return nil + } + + content, err := u.system.ReadFile(path) + if err != nil { + return err + } + + f, err := modfile.Parse(path, content, nil) + if err != nil { + return err + } + + for _, rep := range f.Replace { + // Only process modules from our org/repo that have local replaces + if strings.HasPrefix(rep.Old.Path, orgPrefix) && + isLocalPath(rep.New.Path) && + !seen[rep.Old.Path] { + log.Printf("Found local replace in %s: %s => %s", path, rep.Old.Path, rep.New.Path) + modules = append(modules, rep.Old.Path) + seen[rep.Old.Path] = true + } + } + return nil + }) + + return modules, err +} + func parseModuleVersion(modulePath string) string { ver := module.Version{Path: modulePath} re := regexp.MustCompile(`/v(\d+)$`) @@ -135,4 +216,10 @@ func parseModuleVersion(modulePath string) string { return match[1] } return "0" +} + +func isLocalPath(path string) bool { + return path == "." || path == ".." || + strings.HasPrefix(path, "./") || + strings.HasPrefix(path, "../") } \ No newline at end of file diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index b9d4fd9758e..37f0b4b44f1 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -10,6 +10,8 @@ import ( type mockGitOperator struct { sha string commitDate time.Time + org string + repo string err error } @@ -27,9 +29,19 @@ func (m *mockGitOperator) GetCommitDate(sha string) (time.Time, error) { return m.commitDate, nil } +func (m *mockGitOperator) GetRepoInfo(remote string) (org, repo string, err error) { + if m.err != nil { + return "", "", m.err + } + if m.org == "" && m.repo == "" { + return "smartcontractkit", "chainlink", nil // Default values for tests + } + return m.org, m.repo, nil +} + type mockSystemOperator struct { files map[string][]byte - walkFn func(path string, isDir bool) error + walkFn func(root string, fn func(path string, isDir bool) error) error // Update signature commands []string err error } @@ -61,7 +73,7 @@ func (m *mockSystemOperator) WriteFile(path string, data []byte, perm uint32) er func (m *mockSystemOperator) Walk(root string, fn func(path string, isDir bool) error) error { if m.walkFn != nil { - return m.walkFn(root, true) + return m.walkFn(root, fn) // Pass through the callback } return nil } @@ -94,16 +106,15 @@ func TestUpdater_Run(t *testing.T) { RootPath: ".", }, gitOp: &mockGitOperator{ - sha: "abc123def456", + sha: "abc123def456", // 12 chars commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`module test -require github.com/example/module v0.0.0-20230101000000-123456789abc -`) - m.walkFn = func(path string, isDir bool) error { - return nil +require github.com/example/module v0.0.0-20230101000000-123456789abc`) + m.walkFn = func(root string, fn func(path string, isDir bool) error) error { + return fn("go.mod", false) } return m }(), @@ -117,7 +128,7 @@ require github.com/example/module v0.0.0-20230101000000-123456789abc BranchTrunk: "main", }, gitOp: &mockGitOperator{ - sha: "def456", + sha: "def456789012", // 12 chars commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), }, sysOp: func() *mockSystemOperator { @@ -125,10 +136,12 @@ require github.com/example/module v0.0.0-20230101000000-123456789abc m.files["dir1/go.mod"] = []byte(`require test.com/mod v1.0.0`) m.files["dir2/go.mod"] = []byte(`require test.com/mod v1.0.0`) paths := []string{"dir1/go.mod", "dir2/go.mod"} - i := 0 - m.walkFn = func(path string, isDir bool) error { - if i < len(paths) { - return nil + var i int + m.walkFn = func(root string, fn func(path string, isDir bool) error) error { + for ; i < len(paths); i++ { + if err := fn(paths[i], false); err != nil { + return err + } } return nil } @@ -144,12 +157,70 @@ require github.com/example/module v0.0.0-20230101000000-123456789abc BranchTrunk: "main", }, gitOp: &mockGitOperator{ - sha: "abc123", + sha: "abc123def456", // 12 chars commitDate: time.Now(), }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`require test.com/mod/v2 v2.0.0`) + m.walkFn = func(root string, fn func(path string, isDir bool) error) error { + return fn("go.mod", false) + } + return m + }(), + wantErr: false, + }, + { + name: "finds and updates org modules with local replaces", + config: &Config{ + UpdateOrgModules: true, + RepoRemote: "origin", + BranchTrunk: "main", + OrgName: "smartcontractkit", + RepoName: "chainlink", + }, + gitOp: &mockGitOperator{ + sha: "abc123def456", // 12 chars + commitDate: time.Now(), + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + // Root go.mod for repo detection + m.files["go.mod"] = []byte(`module github.com/smartcontractkit/chainlink/v2`) + // Module with local replace + m.files["core/go.mod"] = []byte(` +module test +require ( + github.com/smartcontractkit/chainlink/v2 v2.0.0 +) +replace github.com/smartcontractkit/chainlink/v2 => ../ +`) + return m + }(), + wantErr: false, + }, + { + name: "skips non-org modules with local replaces", + config: &Config{ + UpdateOrgModules: true, + RepoRemote: "origin", + BranchTrunk: "main", + OrgName: "smartcontractkit", + RepoName: "chainlink", + }, + gitOp: &mockGitOperator{ + sha: "abc123", + commitDate: time.Now(), + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(` +module test +require ( + github.com/other/repo v1.0.0 +) +replace github.com/other/repo => ../other +`) return m }(), wantErr: false, @@ -175,9 +246,12 @@ require ( mod1.com v1.0.0 mod2.com v1.0.0 )`) + sysOp.walkFn = func(root string, fn func(path string, isDir bool) error) error { + return fn("go.mod", false) + } gitOp := &mockGitOperator{ - sha: "abc123", + sha: "abc123def456", // 12 chars commitDate: time.Now(), } @@ -191,4 +265,44 @@ require ( if err := u.Run(); err != nil { t.Errorf("unexpected error: %v", err) } +} + +func TestUpdater_FindLocalReplaceModules(t *testing.T) { + sysOp := newMockSystemOperator() + sysOp.files["go.mod"] = []byte(` +module test +require ( + github.com/smartcontractkit/chainlink/v2 v2.0.0 + github.com/other/repo v1.0.0 +) +replace ( + github.com/smartcontractkit/chainlink/v2 => ../ + github.com/other/repo => ../other +)`) + + // Setup Walk to properly handle the callback + sysOp.walkFn = func(root string, fn func(path string, isDir bool) error) error { + return fn("go.mod", false) // Call the callback with our test file + } + + cfg := &Config{ + UpdateOrgModules: true, + OrgName: "smartcontractkit", + RepoName: "chainlink", + } + + u := New(&mockGitOperator{}, sysOp, cfg) + modules, err := u.findLocalReplaceModules(".") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if len(modules) != 1 { + t.Errorf("expected 1 module, got %d", len(modules)) + return + } + if modules[0] != "github.com/smartcontractkit/chainlink/v2" { + t.Errorf("expected chainlink module, got %s", modules[0]) + } } \ No newline at end of file From cb70261de40d3165afbd1fefea2bd30deacde96a Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:04:53 -0500 Subject: [PATCH 06/47] Update make target to auto update locally replaced modules --- GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 3c80f02cbb3..e334967133d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -32,7 +32,7 @@ gomodtidy: gomods ## Run go mod tidy on all modules. .PHONY: gomodrequiredupdater gomodrequiredupdater: ## Update go.mod files containing certain required dependencies to use latest psuedo-versions from trunk. - cd tools/gomod-required-updater && go run cmd/gomod-required-updater/main.go -module github.com/smartcontractkit/chainlink/v2 -root ../.. + cd tools/gomod-required-updater && go run cmd/gomod-required-updater/main.go -update-org-modules -root ../.. .PHONY: docs docs: ## Install and run pkgsite to view Go docs From 87e940f6646162960c960b69b2ee9e3b35949427 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:06:27 -0500 Subject: [PATCH 07/47] Run go fmt --- .../cmd/gomod-required-updater/main.go | 2 +- .../internal/updater/config.go | 52 ++-- .../internal/updater/config_test.go | 18 +- .../internal/updater/git_operator.go | 42 +-- .../internal/updater/interfaces.go | 32 +-- .../internal/updater/system_operator.go | 2 +- .../internal/updater/updater.go | 242 +++++++++--------- .../internal/updater/updater_test.go | 84 +++--- 8 files changed, 237 insertions(+), 237 deletions(-) diff --git a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go index 90cb7ffa506..8f5d83c9b16 100644 --- a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go +++ b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go @@ -46,4 +46,4 @@ func main() { if err := u.Run(); err != nil { log.Fatal(err) } -} \ No newline at end of file +} diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go index cc4f3c09e51..304a9ee31f5 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -13,41 +13,41 @@ type TOMLConfig struct { } type Config struct { - ModulesToUpdate []string - RepoRemote string - BranchTrunk string - DryRun bool - ConfigFile string - RootPath string - ShowVersion bool - modulesSource string + ModulesToUpdate []string + RepoRemote string + BranchTrunk string + DryRun bool + ConfigFile string + RootPath string + ShowVersion bool + modulesSource string UpdateOrgModules bool // Update modules from same org/repo with local replaces - OrgName string // GitHub organization name - RepoName string // Repository name + OrgName string // GitHub organization name + RepoName string // Repository name } func (c *Config) Validate() error { - if !c.ShowVersion { - // Skip module validation if using UpdateOrgModules - if !c.UpdateOrgModules && len(c.ModulesToUpdate) == 0 { - return fmt.Errorf("%w: no modules specified to update (use -module flag or config file)", ErrInvalidConfig) - } - if c.RepoRemote == "" { - return fmt.Errorf("%w: repo remote cannot be empty", ErrInvalidConfig) - } - if c.BranchTrunk == "" { - return fmt.Errorf("%w: branch trunk cannot be empty", ErrInvalidConfig) - } - } - return nil + if !c.ShowVersion { + // Skip module validation if using UpdateOrgModules + if !c.UpdateOrgModules && len(c.ModulesToUpdate) == 0 { + return fmt.Errorf("%w: no modules specified to update (use -module flag or config file)", ErrInvalidConfig) + } + if c.RepoRemote == "" { + return fmt.Errorf("%w: repo remote cannot be empty", ErrInvalidConfig) + } + if c.BranchTrunk == "" { + return fmt.Errorf("%w: branch trunk cannot be empty", ErrInvalidConfig) + } + } + return nil } func ParseFlags(args []string, version string) (*Config, error) { flags := flag.NewFlagSet("gomod-required-updater", flag.ContinueOnError) - + cfg := &Config{} var cliModules arrayFlags // Define custom flag type for multiple -module flags - + flags.StringVar(&cfg.RepoRemote, "repo-remote", "origin", "The name of the repo remote") flags.StringVar(&cfg.BranchTrunk, "branch-trunk", "develop", "The name of the trunk branch") flags.BoolVar(&cfg.DryRun, "dry-run", false, "Print what would be done without making changes") @@ -110,4 +110,4 @@ func loadTOMLConfig(path string) ([]string, error) { return nil, fmt.Errorf("failed to decode TOML: %w", err) } return cfg.Modules, nil -} \ No newline at end of file +} diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-required-updater/internal/updater/config_test.go index 77a696d18a7..577c0c9aa2a 100644 --- a/tools/gomod-required-updater/internal/updater/config_test.go +++ b/tools/gomod-required-updater/internal/updater/config_test.go @@ -12,8 +12,8 @@ func TestConfig_Validate(t *testing.T) { name: "valid config", cfg: &Config{ ModulesToUpdate: []string{"test.com/mod"}, - RepoRemote: "origin", - BranchTrunk: "main", + RepoRemote: "origin", + BranchTrunk: "main", }, wantErr: false, }, @@ -36,8 +36,8 @@ func TestConfig_Validate(t *testing.T) { name: "update-org-modules bypasses module validation", cfg: &Config{ UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "main", + RepoRemote: "origin", + BranchTrunk: "main", }, wantErr: false, }, @@ -65,8 +65,8 @@ func TestParseFlags(t *testing.T) { args: []string{"-module", "mod1.com", "-module", "mod2.com"}, wantCfg: &Config{ ModulesToUpdate: []string{"mod1.com", "mod2.com"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", }, wantErr: false, }, @@ -75,8 +75,8 @@ func TestParseFlags(t *testing.T) { args: []string{"-version"}, wantCfg: &Config{ ShowVersion: true, - RepoRemote: "origin", // Default value - BranchTrunk: "develop", // Default value + RepoRemote: "origin", // Default value + BranchTrunk: "develop", // Default value }, wantErr: false, }, @@ -92,7 +92,7 @@ func TestParseFlags(t *testing.T) { if err != nil { return } - + if got.RepoRemote != tt.wantCfg.RepoRemote { t.Errorf("ParseFlags() RepoRemote = %v, want %v", got.RepoRemote, tt.wantCfg.RepoRemote) } diff --git a/tools/gomod-required-updater/internal/updater/git_operator.go b/tools/gomod-required-updater/internal/updater/git_operator.go index e5eb3c31b87..29114534647 100644 --- a/tools/gomod-required-updater/internal/updater/git_operator.go +++ b/tools/gomod-required-updater/internal/updater/git_operator.go @@ -38,7 +38,7 @@ func (g *gitOperator) GetSHA(remote, branch string) (string, error) { } sha := strings.TrimSpace(parts[0]) ref := strings.TrimSpace(parts[1]) - + if ref == "refs/heads/"+branch { log.Printf("Found remote SHA: %s for ref: %s", sha, ref) return sha, nil @@ -57,33 +57,33 @@ func (g *gitOperator) GetCommitDate(sha string) (time.Time, error) { if err != nil { return time.Time{}, fmt.Errorf("failed to get commit date: %w", err) } - + dateStr := strings.TrimSpace(string(output)) t, err := time.Parse(time.RFC3339, dateStr) if err != nil { return time.Time{}, fmt.Errorf("failed to parse date: %w", err) } - + return t, nil } func (g *gitOperator) GetRepoInfo(remote string) (org, repo string, err error) { - cmd := exec.Command("git", "config", "--get", fmt.Sprintf("remote.%s.url", remote)) - output, err := cmd.Output() - if err != nil { - return "", "", fmt.Errorf("failed to get repo info for remote %s: %w", remote, err) - } - - // Handle different URL formats: - // https://github.com/org/repo.git - // git@github.com:org/repo.git - url := strings.TrimSpace(string(output)) - parts := strings.Split(strings.TrimSuffix(url, ".git"), "/") - if len(parts) < 2 { - return "", "", fmt.Errorf("unexpected git URL format from remote %s: %s", remote, url) - } - - repo = parts[len(parts)-1] - org = parts[len(parts)-2] - return org, repo, nil + cmd := exec.Command("git", "config", "--get", fmt.Sprintf("remote.%s.url", remote)) + output, err := cmd.Output() + if err != nil { + return "", "", fmt.Errorf("failed to get repo info for remote %s: %w", remote, err) + } + + // Handle different URL formats: + // https://github.com/org/repo.git + // git@github.com:org/repo.git + url := strings.TrimSpace(string(output)) + parts := strings.Split(strings.TrimSuffix(url, ".git"), "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("unexpected git URL format from remote %s: %s", remote, url) + } + + repo = parts[len(parts)-1] + org = parts[len(parts)-2] + return org, repo, nil } diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index a4fd5fc17c8..42ae0f09bab 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -7,28 +7,28 @@ import ( // Errors var ( - ErrInvalidConfig = fmt.Errorf("invalid configuration") - ErrGitOperation = fmt.Errorf("git operation failed") - ErrFileOperation = fmt.Errorf("file operation failed") + ErrInvalidConfig = fmt.Errorf("invalid configuration") + ErrGitOperation = fmt.Errorf("git operation failed") + ErrFileOperation = fmt.Errorf("file operation failed") ) type GitOperator interface { - GetSHA(remote, branch string) (string, error) - GetCommitDate(sha string) (time.Time, error) - GetRepoInfo(remote string) (org, repo string, err error) + GetSHA(remote, branch string) (string, error) + GetCommitDate(sha string) (time.Time, error) + GetRepoInfo(remote string) (org, repo string, err error) } type SystemOperator interface { - ReadFile(path string) ([]byte, error) - WriteFile(filename string, data []byte, perm uint32) error - Walk(root string, fn func(path string, isDir bool) error) error - Chdir(dir string) error - Getwd() (string, error) - RunCommand(name string, args ...string) error + ReadFile(path string) ([]byte, error) + WriteFile(filename string, data []byte, perm uint32) error + Walk(root string, fn func(path string, isDir bool) error) error + Chdir(dir string) error + Getwd() (string, error) + RunCommand(name string, args ...string) error } type Updater struct { - git GitOperator - system SystemOperator - config *Config -} \ No newline at end of file + git GitOperator + system SystemOperator + config *Config +} diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index 302a0466c2c..f2904e997c6 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -62,4 +62,4 @@ func (so *systemOperator) RunCommand(name string, args ...string) error { return fmt.Errorf("%w: %v", ErrFileOperation, err) } return nil -} \ No newline at end of file +} diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 379d82ac9d6..19441561b00 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -21,7 +21,7 @@ func New(git GitOperator, system SystemOperator, config *Config) *Updater { func (u *Updater) Run() error { log.Printf("Starting update process with remote '%s' and branch '%s'", u.config.RepoRemote, u.config.BranchTrunk) - + absRoot, err := filepath.Abs(u.config.RootPath) if err != nil { return fmt.Errorf("failed to resolve root path: %w", err) @@ -81,26 +81,26 @@ func (u *Updater) findRootModule() (string, error) { func (u *Updater) updateGoMod(path, modulePath, sha string) error { log.Printf("Processing %s for module %s", path, modulePath) - - content, err := u.system.ReadFile(path) - if (err != nil) { - return err - } - - f, err := modfile.Parse(path, content, nil) - if err != nil { - return err - } - - moduleExists := false - var currentVersion string - for _, req := range f.Require { - if req.Mod.Path == modulePath { - moduleExists = true - currentVersion = req.Mod.Version - break - } - } + + content, err := u.system.ReadFile(path) + if err != nil { + return err + } + + f, err := modfile.Parse(path, content, nil) + if err != nil { + return err + } + + moduleExists := false + var currentVersion string + for _, req := range f.Require { + if req.Mod.Path == modulePath { + moduleExists = true + currentVersion = req.Mod.Version + break + } + } // Check for replace directive if updating org modules if u.config.UpdateOrgModules { @@ -113,113 +113,113 @@ func (u *Updater) updateGoMod(path, modulePath, sha string) error { } } - if !moduleExists { - log.Printf("Skipping %s: module %s not found", path, modulePath) - return nil - } - - log.Printf("Current version: %s", currentVersion) - - commitDate, err := u.git.GetCommitDate(sha) - if err != nil { - return fmt.Errorf("failed to get commit date: %w", err) - } - - shortSHA := sha[:12] - versionPrefix := parseModuleVersion(modulePath) - pseudoVersion := module.PseudoVersion("v"+versionPrefix, "", commitDate, shortSHA) - - log.Printf("Updating to version: %s", pseudoVersion) - - if u.config.DryRun { - log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, currentVersion, pseudoVersion) - return nil - } - - if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("failed to add requirement: %w", err) - } - - newContent, err := f.Format() - if err != nil { - return fmt.Errorf("failed to format go.mod: %w", err) - } - - if err := u.system.WriteFile(path, newContent, 0644); err != nil { - return fmt.Errorf("failed to write go.mod: %w", err) - } - - dir := filepath.Dir(path) - origDir, err := u.system.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - - if err := u.system.Chdir(dir); err != nil { - return fmt.Errorf("failed to change to directory %s: %w", dir, err) - } + if !moduleExists { + log.Printf("Skipping %s: module %s not found", path, modulePath) + return nil + } + + log.Printf("Current version: %s", currentVersion) + + commitDate, err := u.git.GetCommitDate(sha) + if err != nil { + return fmt.Errorf("failed to get commit date: %w", err) + } + + shortSHA := sha[:12] + versionPrefix := parseModuleVersion(modulePath) + pseudoVersion := module.PseudoVersion("v"+versionPrefix, "", commitDate, shortSHA) + + log.Printf("Updating to version: %s", pseudoVersion) + + if u.config.DryRun { + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, currentVersion, pseudoVersion) + return nil + } + + if err := f.AddRequire(modulePath, pseudoVersion); err != nil { + return fmt.Errorf("failed to add requirement: %w", err) + } + + newContent, err := f.Format() + if err != nil { + return fmt.Errorf("failed to format go.mod: %w", err) + } + + if err := u.system.WriteFile(path, newContent, 0644); err != nil { + return fmt.Errorf("failed to write go.mod: %w", err) + } + + dir := filepath.Dir(path) + origDir, err := u.system.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + + if err := u.system.Chdir(dir); err != nil { + return fmt.Errorf("failed to change to directory %s: %w", dir, err) + } // Use defer to ensure we always return to the initial directory. - defer func() { - if err := u.system.Chdir(origDir); err != nil { - log.Printf("Warning: failed to return to original directory: %v", err) - } - }() - - log.Printf("Running go mod tidy in %s", dir) - if err := u.system.RunCommand("go", "mod", "tidy"); err != nil { - return fmt.Errorf("go mod tidy failed: %w", err) - } - - return nil + defer func() { + if err := u.system.Chdir(origDir); err != nil { + log.Printf("Warning: failed to return to original directory: %v", err) + } + }() + + log.Printf("Running go mod tidy in %s", dir) + if err := u.system.RunCommand("go", "mod", "tidy"); err != nil { + return fmt.Errorf("go mod tidy failed: %w", err) + } + + return nil } func (u *Updater) findLocalReplaceModules(root string) ([]string, error) { - var modules []string - seen := make(map[string]bool) - orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) - - err := u.system.Walk(root, func(path string, isDir bool) error { - if filepath.Base(path) != "go.mod" { - return nil - } - - content, err := u.system.ReadFile(path) - if err != nil { - return err - } - - f, err := modfile.Parse(path, content, nil) - if err != nil { - return err - } - - for _, rep := range f.Replace { - // Only process modules from our org/repo that have local replaces - if strings.HasPrefix(rep.Old.Path, orgPrefix) && - isLocalPath(rep.New.Path) && - !seen[rep.Old.Path] { - log.Printf("Found local replace in %s: %s => %s", path, rep.Old.Path, rep.New.Path) - modules = append(modules, rep.Old.Path) - seen[rep.Old.Path] = true - } - } - return nil - }) - - return modules, err + var modules []string + seen := make(map[string]bool) + orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) + + err := u.system.Walk(root, func(path string, isDir bool) error { + if filepath.Base(path) != "go.mod" { + return nil + } + + content, err := u.system.ReadFile(path) + if err != nil { + return err + } + + f, err := modfile.Parse(path, content, nil) + if err != nil { + return err + } + + for _, rep := range f.Replace { + // Only process modules from our org/repo that have local replaces + if strings.HasPrefix(rep.Old.Path, orgPrefix) && + isLocalPath(rep.New.Path) && + !seen[rep.Old.Path] { + log.Printf("Found local replace in %s: %s => %s", path, rep.Old.Path, rep.New.Path) + modules = append(modules, rep.Old.Path) + seen[rep.Old.Path] = true + } + } + return nil + }) + + return modules, err } func parseModuleVersion(modulePath string) string { - ver := module.Version{Path: modulePath} - re := regexp.MustCompile(`/v(\d+)$`) - if match := re.FindStringSubmatch(ver.Path); match != nil { - return match[1] - } - return "0" + ver := module.Version{Path: modulePath} + re := regexp.MustCompile(`/v(\d+)$`) + if match := re.FindStringSubmatch(ver.Path); match != nil { + return match[1] + } + return "0" } func isLocalPath(path string) bool { - return path == "." || path == ".." || - strings.HasPrefix(path, "./") || - strings.HasPrefix(path, "../") -} \ No newline at end of file + return path == "." || path == ".." || + strings.HasPrefix(path, "./") || + strings.HasPrefix(path, "../") +} diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index 37f0b4b44f1..2c221fcb079 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -101,9 +101,9 @@ func TestUpdater_Run(t *testing.T) { name: "successful update", config: &Config{ ModulesToUpdate: []string{"github.com/example/module"}, - RepoRemote: "origin", - BranchTrunk: "main", - RootPath: ".", + RepoRemote: "origin", + BranchTrunk: "main", + RootPath: ".", }, gitOp: &mockGitOperator{ sha: "abc123def456", // 12 chars @@ -124,8 +124,8 @@ require github.com/example/module v0.0.0-20230101000000-123456789abc`) name: "handles multiple go.mod files", config: &Config{ ModulesToUpdate: []string{"test.com/mod"}, - RepoRemote: "origin", - BranchTrunk: "main", + RepoRemote: "origin", + BranchTrunk: "main", }, gitOp: &mockGitOperator{ sha: "def456789012", // 12 chars @@ -153,8 +153,8 @@ require github.com/example/module v0.0.0-20230101000000-123456789abc`) name: "handles module with version suffix", config: &Config{ ModulesToUpdate: []string{"test.com/mod/v2"}, - RepoRemote: "origin", - BranchTrunk: "main", + RepoRemote: "origin", + BranchTrunk: "main", }, gitOp: &mockGitOperator{ sha: "abc123def456", // 12 chars @@ -174,10 +174,10 @@ require github.com/example/module v0.0.0-20230101000000-123456789abc`) name: "finds and updates org modules with local replaces", config: &Config{ UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "main", - OrgName: "smartcontractkit", - RepoName: "chainlink", + RepoRemote: "origin", + BranchTrunk: "main", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, gitOp: &mockGitOperator{ sha: "abc123def456", // 12 chars @@ -203,10 +203,10 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ name: "skips non-org modules with local replaces", config: &Config{ UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "main", - OrgName: "smartcontractkit", - RepoName: "chainlink", + RepoRemote: "origin", + BranchTrunk: "main", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, gitOp: &mockGitOperator{ sha: "abc123", @@ -257,8 +257,8 @@ require ( cfg := &Config{ ModulesToUpdate: []string{"mod1.com", "mod2.com"}, - RepoRemote: "origin", - BranchTrunk: "main", + RepoRemote: "origin", + BranchTrunk: "main", } u := New(gitOp, sysOp, cfg) @@ -268,8 +268,8 @@ require ( } func TestUpdater_FindLocalReplaceModules(t *testing.T) { - sysOp := newMockSystemOperator() - sysOp.files["go.mod"] = []byte(` + sysOp := newMockSystemOperator() + sysOp.files["go.mod"] = []byte(` module test require ( github.com/smartcontractkit/chainlink/v2 v2.0.0 @@ -280,29 +280,29 @@ replace ( github.com/other/repo => ../other )`) - // Setup Walk to properly handle the callback - sysOp.walkFn = func(root string, fn func(path string, isDir bool) error) error { - return fn("go.mod", false) // Call the callback with our test file - } + // Setup Walk to properly handle the callback + sysOp.walkFn = func(root string, fn func(path string, isDir bool) error) error { + return fn("go.mod", false) // Call the callback with our test file + } - cfg := &Config{ - UpdateOrgModules: true, - OrgName: "smartcontractkit", - RepoName: "chainlink", - } + cfg := &Config{ + UpdateOrgModules: true, + OrgName: "smartcontractkit", + RepoName: "chainlink", + } - u := New(&mockGitOperator{}, sysOp, cfg) - modules, err := u.findLocalReplaceModules(".") - if err != nil { - t.Errorf("unexpected error: %v", err) - return - } + u := New(&mockGitOperator{}, sysOp, cfg) + modules, err := u.findLocalReplaceModules(".") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } - if len(modules) != 1 { - t.Errorf("expected 1 module, got %d", len(modules)) - return - } - if modules[0] != "github.com/smartcontractkit/chainlink/v2" { - t.Errorf("expected chainlink module, got %s", modules[0]) - } -} \ No newline at end of file + if len(modules) != 1 { + t.Errorf("expected 1 module, got %d", len(modules)) + return + } + if modules[0] != "github.com/smartcontractkit/chainlink/v2" { + t.Errorf("expected chainlink module, got %s", modules[0]) + } +} From 5b269ca65244eb5dabdadcb2d8d342b35d10359c Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:22:01 -0500 Subject: [PATCH 08/47] Add ci workflow --- .../workflows/ci-gomod-required-updater.yml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/ci-gomod-required-updater.yml diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml new file mode 100644 index 00000000000..49818403e70 --- /dev/null +++ b/.github/workflows/ci-gomod-required-updater.yml @@ -0,0 +1,41 @@ +name: gomod-required-updater + +on: + merge_group: + pull_request: + +jobs: + lint: + # We don't directly merge dependabot PRs, so let's not waste the resources + if: ${{ (github.event_name == 'pull_request' || github.event_name == 'schedule') && github.actor != 'dependabot[bot]' }} + runs-on: ubuntu-latest + permissions: + # For golangci-lint-actions to annotate code in the PR. + checks: write + contents: read + # For golangci-lint-action's `only-new-issues` option. + pull-requests: read + steps: + - uses: actions/checkout@v4.2.1 + - name: Golang Lint + uses: ./.github/actions/golangci-lint + with: + id: lint + name: lint + go-directory: tools/gomod-required-updater + go-version-file: tools/gomod-required-updater/go.mod + go-module-file: tools/gomod-required-updater/go.sum + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.1 + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version-file: tools/gomod-required-updater/go.mod + go-module-file: tools/gomod-required-updater/go.sum + - name: Run Tests + shell: bash + working-directory: tools/gomod-required-updater + run: go test ./... From 71655148c9d3a7762baeb6aa4180c5b84fed121b Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:47:32 -0500 Subject: [PATCH 09/47] Add test for config sourcing --- .../internal/updater/config.go | 33 +++++++++------- .../internal/updater/config_test.go | 39 +++++++++++++++++++ 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go index 304a9ee31f5..86a8c004be6 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -61,21 +61,15 @@ func ParseFlags(args []string, version string) (*Config, error) { return nil, err } - // CLI modules take precedence - if len(cliModules) > 0 { - cfg.ModulesToUpdate = cliModules - cfg.modulesSource = "command line" - } else if cfg.ConfigFile != "" { - // Only load config file if no CLI modules specified - modules, err := loadTOMLConfig(cfg.ConfigFile) - if err != nil { - return nil, fmt.Errorf("failed to load config: %w", err) + // Validate flag combinations + if cfg.UpdateOrgModules { + if len(cliModules) > 0 { + return nil, fmt.Errorf("%w: -module flag cannot be used with -update-org-modules", ErrInvalidConfig) + } + if cfg.ConfigFile != "" { + return nil, fmt.Errorf("%w: -config flag cannot be used with -update-org-modules", ErrInvalidConfig) } - cfg.ModulesToUpdate = modules - cfg.modulesSource = "config file" - } - if cfg.UpdateOrgModules { gitOp := NewGitOperator() org, repo, err := gitOp.GetRepoInfo(cfg.RepoRemote) if err != nil { @@ -83,6 +77,19 @@ func ParseFlags(args []string, version string) (*Config, error) { } cfg.OrgName = org cfg.RepoName = repo + } else { + // Existing module source precedence logic + if len(cliModules) > 0 { + cfg.ModulesToUpdate = cliModules + cfg.modulesSource = "command line" + } else if cfg.ConfigFile != "" { + modules, err := loadTOMLConfig(cfg.ConfigFile) + if err != nil { + return nil, fmt.Errorf("failed to load config: %w", err) + } + cfg.ModulesToUpdate = modules + cfg.modulesSource = "config file" + } } if err := cfg.Validate(); err != nil { diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-required-updater/internal/updater/config_test.go index 577c0c9aa2a..7ffe9e9882d 100644 --- a/tools/gomod-required-updater/internal/updater/config_test.go +++ b/tools/gomod-required-updater/internal/updater/config_test.go @@ -80,6 +80,45 @@ func TestParseFlags(t *testing.T) { }, wantErr: false, }, + { + name: "module flag takes precedence over config file", + args: []string{ + "-module", "cli.mod", + "-config", "testdata/modules.toml", + }, + wantCfg: &Config{ + ModulesToUpdate: []string{"cli.mod"}, // Only CLI module, config file ignored + RepoRemote: "origin", + BranchTrunk: "develop", + }, + wantErr: false, + }, + { + name: "update-org-modules works alone", + args: []string{"-update-org-modules"}, + wantCfg: &Config{ + UpdateOrgModules: true, + RepoRemote: "origin", + BranchTrunk: "develop", + }, + wantErr: false, + }, + { + name: "module flag disallowed with update-org-modules", + args: []string{ + "-module", "cli.mod", + "-update-org-modules", + }, + wantErr: true, + }, + { + name: "config file disallowed with update-org-modules", + args: []string{ + "-config", "testdata/modules.toml", + "-update-org-modules", + }, + wantErr: true, + }, } for _, tt := range tests { From 979b0c17a6e06062333b4a04f5ad93ea129e5019 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:51:03 -0500 Subject: [PATCH 10/47] Add build step to CI --- .github/workflows/ci-gomod-required-updater.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml index 49818403e70..9e45a8bfee3 100644 --- a/.github/workflows/ci-gomod-required-updater.yml +++ b/.github/workflows/ci-gomod-required-updater.yml @@ -39,3 +39,16 @@ jobs: shell: bash working-directory: tools/gomod-required-updater run: go test ./... + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.1 + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version-file: tools/gomod-required-updater/go.mod + go-module-file: tools/gomod-required-updater/go.sum + - name: Build + working-directory: tools/gomod-required-updater + run: go build -o gomod-required-updater From 99ad50b391636321f4960db9e6db4184fb114ff1 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:29:52 -0500 Subject: [PATCH 11/47] Make it gomods compatible and not a separate module --- GNUmakefile | 10 +- tools/gomod-required-updater/README.md | 11 +- .../cmd/gomod-required-updater/main.go | 13 +- tools/gomod-required-updater/go.mod | 9 - tools/gomod-required-updater/go.sum | 4 - .../internal/updater/config.go | 113 ++--------- .../internal/updater/config_test.go | 79 ++------ .../internal/updater/interfaces.go | 9 - .../internal/updater/system_operator.go | 36 +--- .../internal/updater/updater.go | 187 +++++++----------- .../internal/updater/updater_test.go | 183 +++-------------- 11 files changed, 130 insertions(+), 524 deletions(-) delete mode 100644 tools/gomod-required-updater/go.mod delete mode 100644 tools/gomod-required-updater/go.sum diff --git a/GNUmakefile b/GNUmakefile index e334967133d..3c6591aa037 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -30,10 +30,6 @@ gomod: ## Ensure chainlink's go dependencies are installed. gomodtidy: gomods ## Run go mod tidy on all modules. gomods tidy -.PHONY: gomodrequiredupdater -gomodrequiredupdater: ## Update go.mod files containing certain required dependencies to use latest psuedo-versions from trunk. - cd tools/gomod-required-updater && go run cmd/gomod-required-updater/main.go -update-org-modules -root ../.. - .PHONY: docs docs: ## Install and run pkgsite to view Go docs go install golang.org/x/pkgsite/cmd/pkgsite@latest @@ -148,6 +144,12 @@ presubmit: ## Format go files and imports. gomods: ## Install gomods go install github.com/jmank88/gomods@v0.1.4 +.PHONY: gomodrequiredupdater +gomodrequiredupdater: ## Install and run gomod-required-updater + go install ./tools/gomod-required-updater/cmd/gomod-required-updater + # TODO: -u is the ideal flag for this with gomods but seg faults. + gomods -w gomod-required-updater + .PHONY: mockery mockery: $(mockery) ## Install mockery. go install github.com/vektra/mockery/v2@v2.46.3 diff --git a/tools/gomod-required-updater/README.md b/tools/gomod-required-updater/README.md index dbd89ef5cc0..24679131966 100644 --- a/tools/gomod-required-updater/README.md +++ b/tools/gomod-required-updater/README.md @@ -4,11 +4,8 @@ Updates required module versions in go.mod files to match the latest git SHA fro ## Features -- Update modules to latest SHA from specified branch - Auto-detect and update modules with local replace directives -- Update multiple modules at once - Preview changes with dry run mode -- Configure via TOML or command line flags ## Configuration @@ -24,16 +21,10 @@ modules = [ Command Line Flags: ```shell -Required (one of): - -module Module to update (can be specified multiple times) - -config Path to TOML config file - -update-org-modules Auto-detect and update modules with local replaces - Optional: -repo-remote Git remote to use (default: origin) -branch-trunk Branch to get SHA from (default: develop) - -root Root path for searching go.mod files - -dry-run Preview changes without applying them + -dry-run Preview changes without applying them (default: false) ``` ## Installation diff --git a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go index 8f5d83c9b16..f2988f74a15 100644 --- a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go +++ b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go @@ -5,24 +5,15 @@ import ( "log" "os" - "github.com/smartcontractkit/chainlink/tools/gomod-required-updater/internal/updater" + "github.com/smartcontractkit/chainlink/v2/tools/gomod-required-updater/internal/updater" ) var version = "dev" var usage = `gomod-required-updater version %s Usage: + cd /path/to/go/module gomod-required-updater [flags] - -Examples: - # Update modules specified in config file - gomod-required-updater -config modules.toml - - # Update specific modules - gomod-required-updater -module github.com/org/repo1 -module github.com/org/repo2 - - # Dry run with specific modules - gomod-required-updater -dry-run -module github.com/org/repo1 ` func main() { diff --git a/tools/gomod-required-updater/go.mod b/tools/gomod-required-updater/go.mod deleted file mode 100644 index b6caaedfb33..00000000000 --- a/tools/gomod-required-updater/go.mod +++ /dev/null @@ -1,9 +0,0 @@ -module github.com/smartcontractkit/chainlink/tools/gomod-required-updater - -go 1.23 - -require ( - github.com/BurntSushi/toml v1.4.0 - golang.org/x/mod v0.22.0 - -) diff --git a/tools/gomod-required-updater/go.sum b/tools/gomod-required-updater/go.sum deleted file mode 100644 index c568698e71a..00000000000 --- a/tools/gomod-required-updater/go.sum +++ /dev/null @@ -1,4 +0,0 @@ -github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go index 86a8c004be6..a77e20fe840 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -2,119 +2,40 @@ package updater import ( "flag" - "fmt" - "strings" - - "github.com/BurntSushi/toml" ) -type TOMLConfig struct { - Modules []string `toml:"modules"` -} - type Config struct { - ModulesToUpdate []string - RepoRemote string - BranchTrunk string - DryRun bool - ConfigFile string - RootPath string - ShowVersion bool - modulesSource string - UpdateOrgModules bool // Update modules from same org/repo with local replaces - OrgName string // GitHub organization name - RepoName string // Repository name -} - -func (c *Config) Validate() error { - if !c.ShowVersion { - // Skip module validation if using UpdateOrgModules - if !c.UpdateOrgModules && len(c.ModulesToUpdate) == 0 { - return fmt.Errorf("%w: no modules specified to update (use -module flag or config file)", ErrInvalidConfig) - } - if c.RepoRemote == "" { - return fmt.Errorf("%w: repo remote cannot be empty", ErrInvalidConfig) - } - if c.BranchTrunk == "" { - return fmt.Errorf("%w: branch trunk cannot be empty", ErrInvalidConfig) - } - } - return nil + ModulesToUpdate []string + RepoRemote string + BranchTrunk string + DryRun bool + ShowVersion bool + OrgName string // Set during runtime + RepoName string // Set during runtime } func ParseFlags(args []string, version string) (*Config, error) { flags := flag.NewFlagSet("gomod-required-updater", flag.ContinueOnError) - cfg := &Config{} - var cliModules arrayFlags // Define custom flag type for multiple -module flags + cfg := &Config{ + ModulesToUpdate: make([]string, 0), + } - flags.StringVar(&cfg.RepoRemote, "repo-remote", "origin", "The name of the repo remote") - flags.StringVar(&cfg.BranchTrunk, "branch-trunk", "develop", "The name of the trunk branch") - flags.BoolVar(&cfg.DryRun, "dry-run", false, "Print what would be done without making changes") - flags.StringVar(&cfg.ConfigFile, "config", "", "Path to TOML config file (optional if using -module)") - flags.StringVar(&cfg.RootPath, "root", ".", "Root path to start scanning for go.mod files") + flags.StringVar(&cfg.RepoRemote, "repo-remote", "origin", "Git remote to use") + flags.StringVar(&cfg.BranchTrunk, "branch-trunk", "develop", "Branch to get SHA from") + flags.BoolVar(&cfg.DryRun, "dry-run", false, "Preview changes without applying them") flags.BoolVar(&cfg.ShowVersion, "version", false, "Show version information") - flags.Var(&cliModules, "module", "Module to update (can be specified multiple times)") - flags.BoolVar(&cfg.UpdateOrgModules, "update-org-modules", false, "Update modules from same org/repo that have local replace directives") if err := flags.Parse(args); err != nil { return nil, err } - // Validate flag combinations - if cfg.UpdateOrgModules { - if len(cliModules) > 0 { - return nil, fmt.Errorf("%w: -module flag cannot be used with -update-org-modules", ErrInvalidConfig) - } - if cfg.ConfigFile != "" { - return nil, fmt.Errorf("%w: -config flag cannot be used with -update-org-modules", ErrInvalidConfig) - } - - gitOp := NewGitOperator() - org, repo, err := gitOp.GetRepoInfo(cfg.RepoRemote) - if err != nil { - return nil, fmt.Errorf("failed to get repo info: %w", err) - } - cfg.OrgName = org - cfg.RepoName = repo - } else { - // Existing module source precedence logic - if len(cliModules) > 0 { - cfg.ModulesToUpdate = cliModules - cfg.modulesSource = "command line" - } else if cfg.ConfigFile != "" { - modules, err := loadTOMLConfig(cfg.ConfigFile) - if err != nil { - return nil, fmt.Errorf("failed to load config: %w", err) - } - cfg.ModulesToUpdate = modules - cfg.modulesSource = "config file" - } - } - - if err := cfg.Validate(); err != nil { - return nil, err - } - return cfg, nil } -// arrayFlags allows for repeated flag values -type arrayFlags []string - -func (i *arrayFlags) String() string { - return strings.Join(*i, ", ") -} - -func (i *arrayFlags) Set(value string) error { - *i = append(*i, value) - return nil -} - -func loadTOMLConfig(path string) ([]string, error) { - var cfg TOMLConfig - if _, err := toml.DecodeFile(path, &cfg); err != nil { - return nil, fmt.Errorf("failed to decode TOML: %w", err) +func (c *Config) Validate() error { + if c.ShowVersion { + return nil } - return cfg.Modules, nil + return nil } diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-required-updater/internal/updater/config_test.go index 7ffe9e9882d..b46ea0a9c18 100644 --- a/tools/gomod-required-updater/internal/updater/config_test.go +++ b/tools/gomod-required-updater/internal/updater/config_test.go @@ -10,20 +10,11 @@ func TestConfig_Validate(t *testing.T) { }{ { name: "valid config", - cfg: &Config{ - ModulesToUpdate: []string{"test.com/mod"}, - RepoRemote: "origin", - BranchTrunk: "main", - }, - wantErr: false, - }, - { - name: "missing modules", cfg: &Config{ RepoRemote: "origin", BranchTrunk: "main", }, - wantErr: true, + wantErr: false, }, { name: "version flag bypasses validation", @@ -32,15 +23,6 @@ func TestConfig_Validate(t *testing.T) { }, wantErr: false, }, - { - name: "update-org-modules bypasses module validation", - cfg: &Config{ - UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "main", - }, - wantErr: false, - }, } for _, tt := range tests { @@ -61,12 +43,11 @@ func TestParseFlags(t *testing.T) { wantErr bool }{ { - name: "multiple modules via CLI", - args: []string{"-module", "mod1.com", "-module", "mod2.com"}, + name: "default flags", + args: []string{}, wantCfg: &Config{ - ModulesToUpdate: []string{"mod1.com", "mod2.com"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", }, wantErr: false, }, @@ -75,50 +56,20 @@ func TestParseFlags(t *testing.T) { args: []string{"-version"}, wantCfg: &Config{ ShowVersion: true, - RepoRemote: "origin", // Default value - BranchTrunk: "develop", // Default value - }, - wantErr: false, - }, - { - name: "module flag takes precedence over config file", - args: []string{ - "-module", "cli.mod", - "-config", "testdata/modules.toml", - }, - wantCfg: &Config{ - ModulesToUpdate: []string{"cli.mod"}, // Only CLI module, config file ignored - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", }, wantErr: false, }, { - name: "update-org-modules works alone", - args: []string{"-update-org-modules"}, + name: "custom remote and branch", + args: []string{"-repo-remote", "upstream", "-branch-trunk", "main"}, wantCfg: &Config{ - UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "upstream", + BranchTrunk: "main", }, wantErr: false, }, - { - name: "module flag disallowed with update-org-modules", - args: []string{ - "-module", "cli.mod", - "-update-org-modules", - }, - wantErr: true, - }, - { - name: "config file disallowed with update-org-modules", - args: []string{ - "-config", "testdata/modules.toml", - "-update-org-modules", - }, - wantErr: true, - }, } for _, tt := range tests { @@ -141,14 +92,6 @@ func TestParseFlags(t *testing.T) { if got.ShowVersion != tt.wantCfg.ShowVersion { t.Errorf("ParseFlags() ShowVersion = %v, want %v", got.ShowVersion, tt.wantCfg.ShowVersion) } - if len(got.ModulesToUpdate) != len(tt.wantCfg.ModulesToUpdate) { - t.Errorf("ParseFlags() ModulesToUpdate length = %v, want %v", len(got.ModulesToUpdate), len(tt.wantCfg.ModulesToUpdate)) - } - for i, module := range got.ModulesToUpdate { - if module != tt.wantCfg.ModulesToUpdate[i] { - t.Errorf("ParseFlags() ModulesToUpdate[%d] = %v, want %v", i, module, tt.wantCfg.ModulesToUpdate[i]) - } - } }) } } diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index 42ae0f09bab..32671c72f2a 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -18,15 +18,6 @@ type GitOperator interface { GetRepoInfo(remote string) (org, repo string, err error) } -type SystemOperator interface { - ReadFile(path string) ([]byte, error) - WriteFile(filename string, data []byte, perm uint32) error - Walk(root string, fn func(path string, isDir bool) error) error - Chdir(dir string) error - Getwd() (string, error) - RunCommand(name string, args ...string) error -} - type Updater struct { git GitOperator system SystemOperator diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index f2904e997c6..d0393398a8b 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -1,14 +1,15 @@ package updater import ( - "fmt" "io" - "io/fs" "os" - "os/exec" - "path/filepath" ) +type SystemOperator interface { + ReadFile(path string) ([]byte, error) + WriteFile(filename string, data []byte, perm uint32) error +} + type systemOperator struct { stdout io.Writer stderr io.Writer @@ -36,30 +37,3 @@ func (so *systemOperator) ReadFile(path string) ([]byte, error) { func (so *systemOperator) WriteFile(filename string, data []byte, perm uint32) error { return os.WriteFile(filename, data, os.FileMode(perm)) } - -func (so *systemOperator) Walk(root string, fn func(path string, isDir bool) error) error { - return filepath.Walk(root, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - return fn(path, info.IsDir()) - }) -} - -func (so *systemOperator) Chdir(dir string) error { - return os.Chdir(dir) -} - -func (so *systemOperator) Getwd() (string, error) { - return os.Getwd() -} - -func (so *systemOperator) RunCommand(name string, args ...string) error { - cmd := exec.Command(name, args...) - cmd.Stdout = so.stdout - cmd.Stderr = so.stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("%w: %v", ErrFileOperation, err) - } - return nil -} diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 19441561b00..6fde5a174b1 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -3,7 +3,6 @@ package updater import ( "fmt" "log" - "path/filepath" "regexp" "strings" @@ -22,27 +21,22 @@ func New(git GitOperator, system SystemOperator, config *Config) *Updater { func (u *Updater) Run() error { log.Printf("Starting update process with remote '%s' and branch '%s'", u.config.RepoRemote, u.config.BranchTrunk) - absRoot, err := filepath.Abs(u.config.RootPath) + // Get org and repo info for finding local modules + org, repo, err := u.git.GetRepoInfo(u.config.RepoRemote) if err != nil { - return fmt.Errorf("failed to resolve root path: %w", err) + return fmt.Errorf("failed to get repo info: %w", err) } + u.config.OrgName = org + u.config.RepoName = repo - // If org modules update is enabled, collect all modules with local replaces - if u.config.UpdateOrgModules { - modulesToAdd, err := u.findLocalReplaceModules(absRoot) - if err != nil { - return fmt.Errorf("failed to find local replace modules: %w", err) - } - if len(modulesToAdd) > 0 { - log.Printf("Found %d modules with local replace directives", len(modulesToAdd)) - u.config.ModulesToUpdate = append(u.config.ModulesToUpdate, modulesToAdd...) - } - - rootMod, err := u.findRootModule() - if err != nil { - return fmt.Errorf("failed to find root module: %w", err) - } - u.config.ModulesToUpdate = append(u.config.ModulesToUpdate, rootMod) + // Find modules to update + modulesToAdd, err := u.findLocalReplaceModules() + if err != nil { + return fmt.Errorf("failed to find local replace modules: %w", err) + } + if len(modulesToAdd) > 0 { + log.Printf("Found %d modules with local replace directives", len(modulesToAdd)) + u.config.ModulesToUpdate = append(u.config.ModulesToUpdate, modulesToAdd...) } // Get SHA after collecting modules @@ -53,35 +47,15 @@ func (u *Updater) Run() error { } log.Printf("Using SHA: %s", sha) - return u.system.Walk(absRoot, func(path string, isDir bool) error { - if filepath.Base(path) == "go.mod" { - for _, module := range u.config.ModulesToUpdate { - if err := u.updateGoMod(path, module, sha); err != nil { - return fmt.Errorf("error updating %s: %w", path, err) - } - } - } - return nil - }) -} - -func (u *Updater) findRootModule() (string, error) { - content, err := u.system.ReadFile("go.mod") - if err != nil { - return "", fmt.Errorf("failed to read root go.mod: %w", err) + // Update go.mod in current directory + if err := u.updateGoMod("go.mod", sha); err != nil { + return fmt.Errorf("error updating go.mod: %w", err) } - f, err := modfile.Parse("go.mod", content, nil) - if err != nil { - return "", fmt.Errorf("failed to parse root go.mod: %w", err) - } - - return f.Module.Mod.Path, nil + return nil } -func (u *Updater) updateGoMod(path, modulePath, sha string) error { - log.Printf("Processing %s for module %s", path, modulePath) - +func (u *Updater) updateGoMod(path string, sha string) error { content, err := u.system.ReadFile(path) if err != nil { return err @@ -92,18 +66,18 @@ func (u *Updater) updateGoMod(path, modulePath, sha string) error { return err } - moduleExists := false - var currentVersion string - for _, req := range f.Require { - if req.Mod.Path == modulePath { - moduleExists = true - currentVersion = req.Mod.Version - break + for _, modulePath := range u.config.ModulesToUpdate { + moduleExists := false + var currentVersion string + for _, req := range f.Require { + if req.Mod.Path == modulePath { + moduleExists = true + currentVersion = req.Mod.Version + break + } } - } - // Check for replace directive if updating org modules - if u.config.UpdateOrgModules { + // Check for replace directive for _, rep := range f.Replace { if rep.Old.Path == modulePath && isLocalPath(rep.New.Path) { log.Printf("Found local replace for %s => %s", modulePath, rep.New.Path) @@ -111,33 +85,32 @@ func (u *Updater) updateGoMod(path, modulePath, sha string) error { break } } - } - if !moduleExists { - log.Printf("Skipping %s: module %s not found", path, modulePath) - return nil - } + if !moduleExists { + continue + } - log.Printf("Current version: %s", currentVersion) + log.Printf("Current version: %s", currentVersion) - commitDate, err := u.git.GetCommitDate(sha) - if err != nil { - return fmt.Errorf("failed to get commit date: %w", err) - } + commitDate, err := u.git.GetCommitDate(sha) + if err != nil { + return fmt.Errorf("failed to get commit date: %w", err) + } - shortSHA := sha[:12] - versionPrefix := parseModuleVersion(modulePath) - pseudoVersion := module.PseudoVersion("v"+versionPrefix, "", commitDate, shortSHA) + shortSHA := sha[:12] + versionPrefix := parseModuleVersion(modulePath) + pseudoVersion := module.PseudoVersion("v"+versionPrefix, "", commitDate, shortSHA) - log.Printf("Updating to version: %s", pseudoVersion) + log.Printf("Updating to version: %s", pseudoVersion) - if u.config.DryRun { - log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, currentVersion, pseudoVersion) - return nil - } + if u.config.DryRun { + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, currentVersion, pseudoVersion) + continue + } - if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("failed to add requirement: %w", err) + if err := f.AddRequire(modulePath, pseudoVersion); err != nil { + return fmt.Errorf("failed to add requirement: %w", err) + } } newContent, err := f.Format() @@ -149,64 +122,36 @@ func (u *Updater) updateGoMod(path, modulePath, sha string) error { return fmt.Errorf("failed to write go.mod: %w", err) } - dir := filepath.Dir(path) - origDir, err := u.system.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - - if err := u.system.Chdir(dir); err != nil { - return fmt.Errorf("failed to change to directory %s: %w", dir, err) - } - // Use defer to ensure we always return to the initial directory. - defer func() { - if err := u.system.Chdir(origDir); err != nil { - log.Printf("Warning: failed to return to original directory: %v", err) - } - }() - - log.Printf("Running go mod tidy in %s", dir) - if err := u.system.RunCommand("go", "mod", "tidy"); err != nil { - return fmt.Errorf("go mod tidy failed: %w", err) - } - return nil } -func (u *Updater) findLocalReplaceModules(root string) ([]string, error) { +func (u *Updater) findLocalReplaceModules() ([]string, error) { var modules []string seen := make(map[string]bool) orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) - err := u.system.Walk(root, func(path string, isDir bool) error { - if filepath.Base(path) != "go.mod" { - return nil - } - - content, err := u.system.ReadFile(path) - if err != nil { - return err - } + content, err := u.system.ReadFile("go.mod") + if err != nil { + return nil, err + } - f, err := modfile.Parse(path, content, nil) - if err != nil { - return err - } + f, err := modfile.Parse("go.mod", content, nil) + if err != nil { + return nil, err + } - for _, rep := range f.Replace { - // Only process modules from our org/repo that have local replaces - if strings.HasPrefix(rep.Old.Path, orgPrefix) && - isLocalPath(rep.New.Path) && - !seen[rep.Old.Path] { - log.Printf("Found local replace in %s: %s => %s", path, rep.Old.Path, rep.New.Path) - modules = append(modules, rep.Old.Path) - seen[rep.Old.Path] = true - } + for _, rep := range f.Replace { + // Only process modules from our org/repo that have local replaces + if strings.HasPrefix(rep.Old.Path, orgPrefix) && + isLocalPath(rep.New.Path) && + !seen[rep.Old.Path] { + log.Printf("Found local replace: %s => %s", rep.Old.Path, rep.New.Path) + modules = append(modules, rep.Old.Path) + seen[rep.Old.Path] = true } - return nil - }) + } - return modules, err + return modules, nil } func parseModuleVersion(modulePath string) string { diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index 2c221fcb079..7a2aa4147bb 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -40,10 +40,8 @@ func (m *mockGitOperator) GetRepoInfo(remote string) (org, repo string, err erro } type mockSystemOperator struct { - files map[string][]byte - walkFn func(root string, fn func(path string, isDir bool) error) error // Update signature - commands []string - err error + files map[string][]byte + err error } func newMockSystemOperator() *mockSystemOperator { @@ -71,23 +69,6 @@ func (m *mockSystemOperator) WriteFile(path string, data []byte, perm uint32) er return nil } -func (m *mockSystemOperator) Walk(root string, fn func(path string, isDir bool) error) error { - if m.walkFn != nil { - return m.walkFn(root, fn) // Pass through the callback - } - return nil -} - -func (m *mockSystemOperator) Chdir(dir string) error { return m.err } -func (m *mockSystemOperator) Getwd() (string, error) { return "/mock/dir", m.err } -func (m *mockSystemOperator) RunCommand(name string, args ...string) error { - if m.err != nil { - return m.err - } - m.commands = append(m.commands, name+" "+fmt.Sprint(args)) - return nil -} - func TestUpdater_Run(t *testing.T) { tests := []struct { name string @@ -95,131 +76,46 @@ func TestUpdater_Run(t *testing.T) { gitOp *mockGitOperator sysOp *mockSystemOperator wantErr bool - wantFile string // Expected file content after update + wantFile string }{ { name: "successful update", config: &Config{ - ModulesToUpdate: []string{"github.com/example/module"}, - RepoRemote: "origin", - BranchTrunk: "main", - RootPath: ".", - }, - gitOp: &mockGitOperator{ - sha: "abc123def456", // 12 chars - commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - }, - sysOp: func() *mockSystemOperator { - m := newMockSystemOperator() - m.files["go.mod"] = []byte(`module test -require github.com/example/module v0.0.0-20230101000000-123456789abc`) - m.walkFn = func(root string, fn func(path string, isDir bool) error) error { - return fn("go.mod", false) - } - return m - }(), - wantErr: false, - }, - { - name: "handles multiple go.mod files", - config: &Config{ - ModulesToUpdate: []string{"test.com/mod"}, - RepoRemote: "origin", - BranchTrunk: "main", + RepoRemote: "origin", + BranchTrunk: "main", }, gitOp: &mockGitOperator{ - sha: "def456789012", // 12 chars + sha: "abc123def456", commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + org: "smartcontractkit", + repo: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() - m.files["dir1/go.mod"] = []byte(`require test.com/mod v1.0.0`) - m.files["dir2/go.mod"] = []byte(`require test.com/mod v1.0.0`) - paths := []string{"dir1/go.mod", "dir2/go.mod"} - var i int - m.walkFn = func(root string, fn func(path string, isDir bool) error) error { - for ; i < len(paths); i++ { - if err := fn(paths[i], false); err != nil { - return err - } - } - return nil - } - return m - }(), - wantErr: false, - }, - { - name: "handles module with version suffix", - config: &Config{ - ModulesToUpdate: []string{"test.com/mod/v2"}, - RepoRemote: "origin", - BranchTrunk: "main", - }, - gitOp: &mockGitOperator{ - sha: "abc123def456", // 12 chars - commitDate: time.Now(), - }, - sysOp: func() *mockSystemOperator { - m := newMockSystemOperator() - m.files["go.mod"] = []byte(`require test.com/mod/v2 v2.0.0`) - m.walkFn = func(root string, fn func(path string, isDir bool) error) error { - return fn("go.mod", false) - } - return m - }(), - wantErr: false, - }, - { - name: "finds and updates org modules with local replaces", - config: &Config{ - UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "main", - OrgName: "smartcontractkit", - RepoName: "chainlink", - }, - gitOp: &mockGitOperator{ - sha: "abc123def456", // 12 chars - commitDate: time.Now(), - }, - sysOp: func() *mockSystemOperator { - m := newMockSystemOperator() - // Root go.mod for repo detection - m.files["go.mod"] = []byte(`module github.com/smartcontractkit/chainlink/v2`) - // Module with local replace - m.files["core/go.mod"] = []byte(` -module test -require ( - github.com/smartcontractkit/chainlink/v2 v2.0.0 -) -replace github.com/smartcontractkit/chainlink/v2 => ../ + m.files["go.mod"] = []byte(`module github.com/smartcontractkit/chainlink/v2 +require test.com/mod v1.0.0 `) return m }(), wantErr: false, }, { - name: "skips non-org modules with local replaces", + name: "handles module with local replace", config: &Config{ - UpdateOrgModules: true, - RepoRemote: "origin", - BranchTrunk: "main", - OrgName: "smartcontractkit", - RepoName: "chainlink", + RepoRemote: "origin", + BranchTrunk: "main", }, gitOp: &mockGitOperator{ - sha: "abc123", + sha: "abc123def456", commitDate: time.Now(), + org: "smartcontractkit", + repo: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() - m.files["go.mod"] = []byte(` -module test -require ( - github.com/other/repo v1.0.0 -) -replace github.com/other/repo => ../other + m.files["go.mod"] = []byte(`module test +require github.com/smartcontractkit/chainlink/v2 v2.0.0 +replace github.com/smartcontractkit/chainlink/v2 => ../ `) return m }(), @@ -238,35 +134,6 @@ replace github.com/other/repo => ../other } } -func TestUpdater_UpdateMultipleModules(t *testing.T) { - sysOp := newMockSystemOperator() - sysOp.files["go.mod"] = []byte(` -module test -require ( - mod1.com v1.0.0 - mod2.com v1.0.0 -)`) - sysOp.walkFn = func(root string, fn func(path string, isDir bool) error) error { - return fn("go.mod", false) - } - - gitOp := &mockGitOperator{ - sha: "abc123def456", // 12 chars - commitDate: time.Now(), - } - - cfg := &Config{ - ModulesToUpdate: []string{"mod1.com", "mod2.com"}, - RepoRemote: "origin", - BranchTrunk: "main", - } - - u := New(gitOp, sysOp, cfg) - if err := u.Run(); err != nil { - t.Errorf("unexpected error: %v", err) - } -} - func TestUpdater_FindLocalReplaceModules(t *testing.T) { sysOp := newMockSystemOperator() sysOp.files["go.mod"] = []byte(` @@ -280,19 +147,13 @@ replace ( github.com/other/repo => ../other )`) - // Setup Walk to properly handle the callback - sysOp.walkFn = func(root string, fn func(path string, isDir bool) error) error { - return fn("go.mod", false) // Call the callback with our test file - } - cfg := &Config{ - UpdateOrgModules: true, - OrgName: "smartcontractkit", - RepoName: "chainlink", + OrgName: "smartcontractkit", + RepoName: "chainlink", } u := New(&mockGitOperator{}, sysOp, cfg) - modules, err := u.findLocalReplaceModules(".") + modules, err := u.findLocalReplaceModules() if err != nil { t.Errorf("unexpected error: %v", err) return From b5dc9fb4344836c3e4d8d15619acbe378f465dba Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:35:38 -0500 Subject: [PATCH 12/47] Remove mocked sys op --- .../internal/updater/system_operator.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index d0393398a8b..7965a1c0297 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -22,14 +22,6 @@ func NewSystemOperator() SystemOperator { } } -// For testing -func NewSystemOperatorWithIO(stdout, stderr io.Writer) SystemOperator { - return &systemOperator{ - stdout: stdout, - stderr: stderr, - } -} - func (so *systemOperator) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } From 683ae996b02a7e3bd52c00feeefb9ba63884a3dc Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:50:27 -0500 Subject: [PATCH 13/47] Only build during dedicated CI workflow. Tests and linting should now be ran from the ci-core workflow. --- .../workflows/ci-gomod-required-updater.yml | 43 +------------------ 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml index 9e45a8bfee3..5108ebbbb25 100644 --- a/.github/workflows/ci-gomod-required-updater.yml +++ b/.github/workflows/ci-gomod-required-updater.yml @@ -5,50 +5,11 @@ on: pull_request: jobs: - lint: - # We don't directly merge dependabot PRs, so let's not waste the resources - if: ${{ (github.event_name == 'pull_request' || github.event_name == 'schedule') && github.actor != 'dependabot[bot]' }} - runs-on: ubuntu-latest - permissions: - # For golangci-lint-actions to annotate code in the PR. - checks: write - contents: read - # For golangci-lint-action's `only-new-issues` option. - pull-requests: read - steps: - - uses: actions/checkout@v4.2.1 - - name: Golang Lint - uses: ./.github/actions/golangci-lint - with: - id: lint - name: lint - go-directory: tools/gomod-required-updater - go-version-file: tools/gomod-required-updater/go.mod - go-module-file: tools/gomod-required-updater/go.sum - - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.2.1 - - name: Setup Go - uses: ./.github/actions/setup-go - with: - go-version-file: tools/gomod-required-updater/go.mod - go-module-file: tools/gomod-required-updater/go.sum - - name: Run Tests - shell: bash - working-directory: tools/gomod-required-updater - run: go test ./... - build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.2.1 - name: Setup Go uses: ./.github/actions/setup-go - with: - go-version-file: tools/gomod-required-updater/go.mod - go-module-file: tools/gomod-required-updater/go.sum - - name: Build - working-directory: tools/gomod-required-updater - run: go build -o gomod-required-updater + - name: Build binary + run: go build ./tools/gomod-required-updater/cmd/gomod-required-updater From 73583675a5cf969787119b881945a822d5724932 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:29:01 -0500 Subject: [PATCH 14/47] Refactor --- GNUmakefile | 4 +- core/scripts/go.mod | 4 +- deployment/go.mod | 2 +- integration-tests/go.mod | 4 +- integration-tests/load/go.mod | 6 +- .../cmd/gomod-required-updater/main.go | 2 +- .../internal/updater/git_operator.go | 89 -------- .../internal/updater/interfaces.go | 19 +- .../internal/updater/module_operator.go | 150 +++++++++++++ .../internal/updater/updater.go | 145 +++++++------ .../internal/updater/updater_test.go | 202 +++++++++++++++--- 11 files changed, 427 insertions(+), 200 deletions(-) delete mode 100644 tools/gomod-required-updater/internal/updater/git_operator.go create mode 100644 tools/gomod-required-updater/internal/updater/module_operator.go diff --git a/GNUmakefile b/GNUmakefile index 3c6591aa037..373fe2bfb32 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -145,10 +145,10 @@ gomods: ## Install gomods go install github.com/jmank88/gomods@v0.1.4 .PHONY: gomodrequiredupdater -gomodrequiredupdater: ## Install and run gomod-required-updater +gomodrequiredupdater: ## Run gomod-required-updater go install ./tools/gomod-required-updater/cmd/gomod-required-updater - # TODO: -u is the ideal flag for this with gomods but seg faults. gomods -w gomod-required-updater + gomods tidy .PHONY: mockery mockery: $(mockery) ## Install mockery. diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 3814af1b709..6bdf9205d8b 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -10,8 +10,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed ) require ( diff --git a/deployment/go.mod b/deployment/go.mod index 34e8ec80961..5bc3a4e0c8a 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -7,7 +7,7 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ // Using a separate inline `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed require ( github.com/Khan/genqlient v0.7.0 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index bb5469aa3de..1359d23b63e 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -10,8 +10,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed ) require ( diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 6424b05a4c0..c578d2a3413 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -12,9 +12,9 @@ replace github.com/smartcontractkit/chainlink/integration-tests => ../ // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d - github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241119120536-03115e80382d - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed ) require ( diff --git a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go index f2988f74a15..acb01779ee9 100644 --- a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go +++ b/tools/gomod-required-updater/cmd/gomod-required-updater/main.go @@ -29,7 +29,7 @@ func main() { } u := updater.New( - updater.NewGitOperator(), + updater.NewModuleOperator(cfg), updater.NewSystemOperator(), cfg, ) diff --git a/tools/gomod-required-updater/internal/updater/git_operator.go b/tools/gomod-required-updater/internal/updater/git_operator.go deleted file mode 100644 index 29114534647..00000000000 --- a/tools/gomod-required-updater/internal/updater/git_operator.go +++ /dev/null @@ -1,89 +0,0 @@ -package updater - -import ( - "fmt" - "log" - "os/exec" - "strings" - "time" -) - -type gitOperator struct{} - -// NewGitOperator creates a new instance of GitOperator -func NewGitOperator() GitOperator { - return &gitOperator{} -} - -// GetSHA returns the SHA of a given branch in a remote repository -func (g *gitOperator) GetSHA(remote, branch string) (string, error) { - // Use the full ref path to get exact match - cmd := exec.Command("git", "ls-remote", remote, "refs/heads/"+branch) - log.Printf("Running git command: git ls-remote %s refs/heads/%s", remote, branch) - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("%w: failed to get remote SHA: %v", ErrGitOperation, err) - } - - lines := strings.Split(strings.TrimSpace(string(output)), "\n") - if len(lines) == 0 || lines[0] == "" { - return "", fmt.Errorf("%w: no SHA found for branch %s in remote %s", ErrGitOperation, branch, remote) - } - - // Parse the first line that exactly matches our ref - for _, line := range lines { - parts := strings.Split(line, "\t") - if len(parts) != 2 { - continue - } - sha := strings.TrimSpace(parts[0]) - ref := strings.TrimSpace(parts[1]) - - if ref == "refs/heads/"+branch { - log.Printf("Found remote SHA: %s for ref: %s", sha, ref) - return sha, nil - } - } - - return "", fmt.Errorf("%w: no exact match found for refs/heads/%s", ErrGitOperation, branch) -} - -// GetCommitDate returns the commit date of a given SHA used for in the go.mod -// psuedo-version for the module. -func (g *gitOperator) GetCommitDate(sha string) (time.Time, error) { - // Get commit date in ISO 8601 time format - cmd := exec.Command("git", "show", "-s", "--format=%cI", sha) - output, err := cmd.Output() - if err != nil { - return time.Time{}, fmt.Errorf("failed to get commit date: %w", err) - } - - dateStr := strings.TrimSpace(string(output)) - t, err := time.Parse(time.RFC3339, dateStr) - if err != nil { - return time.Time{}, fmt.Errorf("failed to parse date: %w", err) - } - - return t, nil -} - -func (g *gitOperator) GetRepoInfo(remote string) (org, repo string, err error) { - cmd := exec.Command("git", "config", "--get", fmt.Sprintf("remote.%s.url", remote)) - output, err := cmd.Output() - if err != nil { - return "", "", fmt.Errorf("failed to get repo info for remote %s: %w", remote, err) - } - - // Handle different URL formats: - // https://github.com/org/repo.git - // git@github.com:org/repo.git - url := strings.TrimSpace(string(output)) - parts := strings.Split(strings.TrimSuffix(url, ".git"), "/") - if len(parts) < 2 { - return "", "", fmt.Errorf("unexpected git URL format from remote %s: %s", remote, url) - } - - repo = parts[len(parts)-1] - org = parts[len(parts)-2] - return org, repo, nil -} diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index 32671c72f2a..20c2cc02d9f 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -3,23 +3,30 @@ package updater import ( "fmt" "time" + + "golang.org/x/mod/module" ) // Errors var ( ErrInvalidConfig = fmt.Errorf("invalid configuration") - ErrGitOperation = fmt.Errorf("git operation failed") + ErrModOperation = fmt.Errorf("module operation failed") ErrFileOperation = fmt.Errorf("file operation failed") ) -type GitOperator interface { - GetSHA(remote, branch string) (string, error) - GetCommitDate(sha string) (time.Time, error) - GetRepoInfo(remote string) (org, repo string, err error) +type ModuleOperator interface { + // GetLatestVersion gets the latest pseudo-version based on current git state + GetLatestVersion(modulePath string) (module.Version, error) + // GetModuleInfo gets version info including timestamp + GetModuleInfo(modulePath string) (module.Version, time.Time, error) + // ParseModulePathParts extracts org and repo from module path + ParseModulePathParts(modulePath string) (org, repo string, err error) + // GetGitInfo gets the latest git SHA and commit time + GetGitInfo(remote, branch string) (sha string, commitTime time.Time, err error) } type Updater struct { - git GitOperator + mod ModuleOperator system SystemOperator config *Config } diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go new file mode 100644 index 00000000000..05c75f7aa25 --- /dev/null +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -0,0 +1,150 @@ +package updater + +import ( + "encoding/json" + "fmt" + "os/exec" + "regexp" + "strings" + "time" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +const ( + majorVersionPattern = `/v\d+$` +) + +// getMajorVersion extracts the major version number from a module path +// Returns "v2" for /v2, "v0" for no version suffix +func getMajorVersion(modulePath string) string { + re := regexp.MustCompile(majorVersionPattern) + if match := re.FindString(modulePath); match != "" { + return "v" + strings.TrimPrefix(match, "/v") + } + return "v0" +} + +type moduleOperator struct { + config *Config +} + +func NewModuleOperator(config *Config) ModuleOperator { + if config.RepoRemote == "" { + config.RepoRemote = "origin" + } + if config.BranchTrunk == "" { + config.BranchTrunk = "develop" + } + return &moduleOperator{ + config: config, + } +} + +func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, error) { + // Get latest SHA + cmd := exec.Command("git", "ls-remote", remote, "refs/heads/"+branch) + out, err := cmd.Output() + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: failed to get SHA: %v", ErrModOperation, err) + } + sha := strings.Split(string(out), "\t")[0] + + // Get commit timestamp + cmd = exec.Command("git", "show", "-s", "--format=%cI", sha) + out, err = cmd.Output() + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: failed to get commit time: %v", ErrModOperation, err) + } + commitTime, err := time.Parse(time.RFC3339, strings.TrimSpace(string(out))) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: failed to parse commit time: %v", ErrModOperation, err) + } + + return sha[:12], commitTime, nil +} + +func (m *moduleOperator) GetLatestVersion(modulePath string) (module.Version, error) { + sha, commitTime, err := m.GetGitInfo(m.config.RepoRemote, m.config.BranchTrunk) + if err != nil { + return module.Version{}, err + } + + majorVer := strings.TrimPrefix(getMajorVersion(modulePath), "v") + pseudoVersion := module.PseudoVersion("v"+majorVer, "", commitTime, sha) + + return module.Version{ + Path: modulePath, + Version: pseudoVersion, + }, nil +} + +func (m *moduleOperator) validateVersion(modulePath, version string) error { + expectedMajor := getMajorVersion(modulePath) + if !strings.HasPrefix(version, expectedMajor) { + return fmt.Errorf("version %q invalid: should be %s, not v0", version, expectedMajor) + } + return nil +} + +func (m *moduleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) error { + for _, req := range modFile.Require { + if strings.HasPrefix(req.Mod.Path, "github.com/smartcontractkit/chainlink") { + // Check if this is a v2+ module path + modVersion := getMajorVersion(req.Mod.Path) + if modVersion != "v0" { + // If module path contains v2+, maintain that version + newVer := strings.Replace(newVersion, "v0", modVersion, 1) + req.Mod.Version = newVer + } else { + // For modules without version in path, use newVersion as-is + req.Mod.Version = newVersion + } + } + } + return nil +} + +type moduleInfo struct { + Path string `json:"Path"` + Version string `json:"Version"` + Time time.Time `json:"Time"` +} + +// GetModuleInfo gets version info including timestamp +func (m *moduleOperator) GetModuleInfo(modulePath string) (module.Version, time.Time, error) { + cmd := exec.Command("go", "list", "-m", "-json", modulePath+"@latest") + out, err := cmd.Output() + if err != nil { + return module.Version{}, time.Time{}, fmt.Errorf("%w: %v", ErrModOperation, err) + } + + var info moduleInfo + if err := json.Unmarshal(out, &info); err != nil { + return module.Version{}, time.Time{}, fmt.Errorf("%w: failed to decode response: %v", ErrModOperation, err) + } + + return module.Version{ + Path: info.Path, + Version: info.Version, + }, info.Time, nil +} + +// ParseModulePathParts extracts org and repo from module path +func (m *moduleOperator) ParseModulePathParts(modulePath string) (org, repo string, err error) { + parts := strings.Split(modulePath, "/") + if len(parts) < 3 { + return "", "", fmt.Errorf("%w: invalid module path format: %s", ErrModOperation, modulePath) + } + + org = parts[1] + repo = parts[2] + + // Strip version suffix if present (e.g., "repo/v2" -> "repo") + if i := strings.Index(repo, "/v"); i != -1 { + repo = repo[:i] + } + + return org, repo, nil +} diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 6fde5a174b1..3b256e1b7ad 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -3,59 +3,78 @@ package updater import ( "fmt" "log" - "regexp" "strings" "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) -func New(git GitOperator, system SystemOperator, config *Config) *Updater { +// New creates a new Updater +func New(mod ModuleOperator, system SystemOperator, config *Config) *Updater { return &Updater{ - git: git, + mod: mod, system: system, config: config, } } +// Run the update process func (u *Updater) Run() error { - log.Printf("Starting update process with remote '%s' and branch '%s'", u.config.RepoRemote, u.config.BranchTrunk) + if len(u.config.ModulesToUpdate) == 0 { + log.Printf("No modules specified, will auto-detect modules with local replace directives") + } else { + log.Printf("Starting update process for modules: %v", u.config.ModulesToUpdate) + } - // Get org and repo info for finding local modules - org, repo, err := u.git.GetRepoInfo(u.config.RepoRemote) + // Get org and repo info from current module first + content, err := u.system.ReadFile("go.mod") if err != nil { - return fmt.Errorf("failed to get repo info: %w", err) + return fmt.Errorf("failed to read go.mod: %w", err) } - u.config.OrgName = org - u.config.RepoName = repo - // Find modules to update - modulesToAdd, err := u.findLocalReplaceModules() + f, err := modfile.Parse("go.mod", content, nil) if err != nil { - return fmt.Errorf("failed to find local replace modules: %w", err) + return fmt.Errorf("failed to parse go.mod: %w", err) } - if len(modulesToAdd) > 0 { - log.Printf("Found %d modules with local replace directives", len(modulesToAdd)) - u.config.ModulesToUpdate = append(u.config.ModulesToUpdate, modulesToAdd...) + + // Get org/repo from the current module path + org, repo, err := u.mod.ParseModulePathParts(f.Module.Mod.Path) + if err != nil { + return fmt.Errorf("failed to get repo info from current module: %w", err) } + u.config.OrgName = org + u.config.RepoName = repo - // Get SHA after collecting modules - log.Printf("Fetching latest SHA from %s/%s", u.config.RepoRemote, u.config.BranchTrunk) - sha, err := u.git.GetSHA(u.config.RepoRemote, u.config.BranchTrunk) + // Find modules to update first if none specified + if len(u.config.ModulesToUpdate) == 0 { + modulesToAdd, err := u.findLocalReplaceModules() + if err != nil { + return fmt.Errorf("failed to find local replace modules: %w", err) + } + if len(modulesToAdd) == 0 { + log.Printf("No modules found to update in %s", f.Module.Mod.Path) + return nil // This is now a non-error case + } + u.config.ModulesToUpdate = modulesToAdd + log.Printf("Found %d modules with local replace directives: %v", len(modulesToAdd), modulesToAdd) + } + + // Get latest version once and apply to all modules + version, err := u.mod.GetLatestVersion(u.config.ModulesToUpdate[0]) if err != nil { - return fmt.Errorf("failed to get SHA: %w", err) + return fmt.Errorf("failed to get latest version: %w", err) } - log.Printf("Using SHA: %s", sha) + log.Printf("Using version: %s for all modules", version.Version) - // Update go.mod in current directory - if err := u.updateGoMod("go.mod", sha); err != nil { + if err := u.updateGoMod("go.mod", version); err != nil { return fmt.Errorf("error updating go.mod: %w", err) } return nil } -func (u *Updater) updateGoMod(path string, sha string) error { +// updateGoMod updates the go.mod file with the new module version +func (u *Updater) updateGoMod(path string, newVersion module.Version) error { content, err := u.system.ReadFile(path) if err != nil { return err @@ -68,20 +87,45 @@ func (u *Updater) updateGoMod(path string, sha string) error { for _, modulePath := range u.config.ModulesToUpdate { moduleExists := false - var currentVersion string + + // Get major version from module path suffix (/v2, /v3, etc) + majorVersion := "v0" + if idx := strings.LastIndex(modulePath, "/v"); idx != -1 { + versionSuffix := modulePath[idx+1:] // get everything after the /v + if _, err := fmt.Sscanf(versionSuffix, "v%d", new(int)); err == nil { + majorVersion = versionSuffix + } + } + + // Get timestamp and commit hash from version string + parts := strings.Split(newVersion.Version, "-") + var timestamp, commitHash string + if len(parts) >= 3 { + timestamp = parts[1] + commitHash = parts[2] + } else { + timestamp = "00000000000000" + commitHash = newVersion.Version // use full version as commit hash if can't parse + } + + // Format the version based on module's major version from path + targetVersion := fmt.Sprintf("%s.0.0-%s-%s", majorVersion, timestamp, commitHash) + if majorVersion == "v0" { + targetVersion = fmt.Sprintf("v0.0.0-%s-%s", timestamp, commitHash) + } + + // Find current version for _, req := range f.Require { if req.Mod.Path == modulePath { moduleExists = true - currentVersion = req.Mod.Version - break - } - } - - // Check for replace directive - for _, rep := range f.Replace { - if rep.Old.Path == modulePath && isLocalPath(rep.New.Path) { - log.Printf("Found local replace for %s => %s", modulePath, rep.New.Path) - moduleExists = true + if u.config.DryRun { + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, targetVersion) + continue + } + + if err := f.AddRequire(modulePath, targetVersion); err != nil { + return fmt.Errorf("failed to add requirement: %w", err) + } break } } @@ -89,28 +133,6 @@ func (u *Updater) updateGoMod(path string, sha string) error { if !moduleExists { continue } - - log.Printf("Current version: %s", currentVersion) - - commitDate, err := u.git.GetCommitDate(sha) - if err != nil { - return fmt.Errorf("failed to get commit date: %w", err) - } - - shortSHA := sha[:12] - versionPrefix := parseModuleVersion(modulePath) - pseudoVersion := module.PseudoVersion("v"+versionPrefix, "", commitDate, shortSHA) - - log.Printf("Updating to version: %s", pseudoVersion) - - if u.config.DryRun { - log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, currentVersion, pseudoVersion) - continue - } - - if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("failed to add requirement: %w", err) - } } newContent, err := f.Format() @@ -125,6 +147,7 @@ func (u *Updater) updateGoMod(path string, sha string) error { return nil } +// findLocalReplaceModules finds modules with local replace directives func (u *Updater) findLocalReplaceModules() ([]string, error) { var modules []string seen := make(map[string]bool) @@ -154,15 +177,7 @@ func (u *Updater) findLocalReplaceModules() ([]string, error) { return modules, nil } -func parseModuleVersion(modulePath string) string { - ver := module.Version{Path: modulePath} - re := regexp.MustCompile(`/v(\d+)$`) - if match := re.FindStringSubmatch(ver.Path); match != nil { - return match[1] - } - return "0" -} - +// isLocalPath checks if the path is a local path func isLocalPath(path string) bool { return path == "." || path == ".." || strings.HasPrefix(path, "./") || diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index 7a2aa4147bb..5a070f1e768 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -4,32 +4,35 @@ import ( "fmt" "testing" "time" + + "golang.org/x/mod/module" ) // Mock implementations -type mockGitOperator struct { - sha string - commitDate time.Time +type mockModuleOperator struct { + version module.Version + updateTime time.Time org string repo string + sha string err error } -func (m *mockGitOperator) GetSHA(remote, branch string) (string, error) { +func (m *mockModuleOperator) GetLatestVersion(modulePath string) (module.Version, error) { if m.err != nil { - return "", m.err + return module.Version{}, m.err } - return m.sha, nil + return m.version, nil } -func (m *mockGitOperator) GetCommitDate(sha string) (time.Time, error) { +func (m *mockModuleOperator) GetModuleInfo(modulePath string) (module.Version, time.Time, error) { if m.err != nil { - return time.Time{}, m.err + return module.Version{}, time.Time{}, m.err } - return m.commitDate, nil + return m.version, m.updateTime, nil } -func (m *mockGitOperator) GetRepoInfo(remote string) (org, repo string, err error) { +func (m *mockModuleOperator) ParseModulePathParts(modulePath string) (org, repo string, err error) { if m.err != nil { return "", "", m.err } @@ -39,6 +42,16 @@ func (m *mockGitOperator) GetRepoInfo(remote string) (org, repo string, err erro return m.org, m.repo, nil } +func (m *mockModuleOperator) GetGitInfo(remote, branch string) (string, time.Time, error) { + if m.err != nil { + return "", time.Time{}, m.err + } + if m.sha == "" { + m.sha = "abcdef123456" // default test SHA + } + return m.sha, m.updateTime, nil +} + type mockSystemOperator struct { files map[string][]byte err error @@ -73,7 +86,7 @@ func TestUpdater_Run(t *testing.T) { tests := []struct { name string config *Config - gitOp *mockGitOperator + modOp *mockModuleOperator sysOp *mockSystemOperator wantErr bool wantFile string @@ -81,19 +94,19 @@ func TestUpdater_Run(t *testing.T) { { name: "successful update", config: &Config{ - RepoRemote: "origin", - BranchTrunk: "main", + ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, }, - gitOp: &mockGitOperator{ - sha: "abc123def456", - commitDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - org: "smartcontractkit", - repo: "chainlink", + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/smartcontractkit/chainlink/v2", + Version: "v2.0.0", + }, + updateTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() - m.files["go.mod"] = []byte(`module github.com/smartcontractkit/chainlink/v2 -require test.com/mod v1.0.0 + m.files["go.mod"] = []byte(`module test +require github.com/smartcontractkit/chainlink/v2 v2.0.0 `) return m }(), @@ -102,14 +115,13 @@ require test.com/mod v1.0.0 { name: "handles module with local replace", config: &Config{ - RepoRemote: "origin", - BranchTrunk: "main", + ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, }, - gitOp: &mockGitOperator{ - sha: "abc123def456", - commitDate: time.Now(), - org: "smartcontractkit", - repo: "chainlink", + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/smartcontractkit/chainlink/v2", + Version: "v2.0.0", + }, }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -121,15 +133,147 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ }(), wantErr: false, }, + { + name: "v1 module update", + config: &Config{ + ModulesToUpdate: []string{"github.com/example/mod"}, + }, + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/example/mod", + Version: "v1.2.3", + }, + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`module test +require github.com/example/mod v1.0.0 +`) + return m + }(), + wantErr: false, + }, + { + name: "updates v2 module with timestamp", + config: &Config{ + ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, + }, + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/smartcontractkit/chainlink/v2", + Version: "v2.0.0-20241122182110-ac7a7395feed", + }, + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`module test +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d +`) + return m + }(), + wantErr: false, + wantFile: `module test + +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed +`, + }, + { + name: "updates v0 module with timestamp", + config: &Config{ + ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/deployment"}, + }, + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/smartcontractkit/chainlink/deployment", + Version: "v2.0.0-20241122182110-ac7a7395feed", + }, + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`module test +require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d +`) + return m + }(), + wantErr: false, + wantFile: `module test + +require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed +`, + }, + { + name: "handles multiple modules with different versions", + config: &Config{ + ModulesToUpdate: []string{ + "github.com/smartcontractkit/chainlink/v2", + "github.com/smartcontractkit/chainlink/deployment", + }, + }, + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/smartcontractkit/chainlink/v2", + Version: "v2.0.0-20241122182110-ac7a7395feed", + }, + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`module test +require ( + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e80382d + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-03115e80382d +) +`) + return m + }(), + wantErr: false, + wantFile: `module test + +require ( + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed +) +`, + }, + { + name: "updates v3 module with timestamp", + config: &Config{ + ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v3"}, + }, + modOp: &mockModuleOperator{ + version: module.Version{ + Path: "github.com/smartcontractkit/chainlink/v3", + Version: "v2.0.0-20241122182110-ac7a7395feed", // Version from main branch + }, + }, + sysOp: func() *mockSystemOperator { + m := newMockSystemOperator() + m.files["go.mod"] = []byte(`module test +require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241119120536-03115e80382d +`) + return m + }(), + wantErr: false, + wantFile: `module test + +require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241122182110-ac7a7395feed +`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - u := New(tt.gitOp, tt.sysOp, tt.config) + u := New(tt.modOp, tt.sysOp, tt.config) err := u.Run() if (err != nil) != tt.wantErr { t.Errorf("Updater.Run() error = %v, wantErr %v", err, tt.wantErr) } + + if tt.wantFile != "" { + got := string(tt.sysOp.files["go.mod"]) + if got != tt.wantFile { + t.Errorf("File content mismatch\nGot:\n%s\nWant:\n%s", got, tt.wantFile) + } + } }) } } @@ -152,7 +296,7 @@ replace ( RepoName: "chainlink", } - u := New(&mockGitOperator{}, sysOp, cfg) + u := New(&mockModuleOperator{}, sysOp, cfg) modules, err := u.findLocalReplaceModules() if err != nil { t.Errorf("unexpected error: %v", err) From e47bdd19fba417c0ef0c726ff5d31f1b355405ac Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:33:18 -0500 Subject: [PATCH 15/47] Remove unused function --- .../internal/updater/module_operator.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go index 05c75f7aa25..41a09249f15 100644 --- a/tools/gomod-required-updater/internal/updater/module_operator.go +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -80,14 +80,6 @@ func (m *moduleOperator) GetLatestVersion(modulePath string) (module.Version, er }, nil } -func (m *moduleOperator) validateVersion(modulePath, version string) error { - expectedMajor := getMajorVersion(modulePath) - if !strings.HasPrefix(version, expectedMajor) { - return fmt.Errorf("version %q invalid: should be %s, not v0", version, expectedMajor) - } - return nil -} - func (m *moduleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) error { for _, req := range modFile.Require { if strings.HasPrefix(req.Mod.Path, "github.com/smartcontractkit/chainlink") { From eb28c011376fbd0e5ad6b004de3770e01b712557 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 16:14:10 -0500 Subject: [PATCH 16/47] Use psuedo version from go lib --- .../internal/updater/updater.go | 59 ++++++++----------- .../internal/updater/updater_test.go | 37 ++++++------ 2 files changed, 43 insertions(+), 53 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 3b256e1b7ad..3a45ec65860 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "strings" + "time" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -59,22 +60,21 @@ func (u *Updater) Run() error { log.Printf("Found %d modules with local replace directives: %v", len(modulesToAdd), modulesToAdd) } - // Get latest version once and apply to all modules - version, err := u.mod.GetLatestVersion(u.config.ModulesToUpdate[0]) + // Get commit info once for all modules + sha, commitTime, err := u.mod.GetGitInfo(u.config.RepoRemote, u.config.BranchTrunk) if err != nil { - return fmt.Errorf("failed to get latest version: %w", err) + return fmt.Errorf("failed to get git info: %w", err) } - log.Printf("Using version: %s for all modules", version.Version) - if err := u.updateGoMod("go.mod", version); err != nil { + if err := u.updateGoMod("go.mod", sha, commitTime); err != nil { return fmt.Errorf("error updating go.mod: %w", err) } return nil } -// updateGoMod updates the go.mod file with the new module version -func (u *Updater) updateGoMod(path string, newVersion module.Version) error { +// updateGoMod updates the go.mod file with new pseudo-versions +func (u *Updater) updateGoMod(path string, sha string, commitTime time.Time) error { content, err := u.system.ReadFile(path) if err != nil { return err @@ -88,42 +88,29 @@ func (u *Updater) updateGoMod(path string, newVersion module.Version) error { for _, modulePath := range u.config.ModulesToUpdate { moduleExists := false - // Get major version from module path suffix (/v2, /v3, etc) - majorVersion := "v0" - if idx := strings.LastIndex(modulePath, "/v"); idx != -1 { - versionSuffix := modulePath[idx+1:] // get everything after the /v - if _, err := fmt.Sscanf(versionSuffix, "v%d", new(int)); err == nil { - majorVersion = versionSuffix - } - } - - // Get timestamp and commit hash from version string - parts := strings.Split(newVersion.Version, "-") - var timestamp, commitHash string - if len(parts) >= 3 { - timestamp = parts[1] - commitHash = parts[2] - } else { - timestamp = "00000000000000" - commitHash = newVersion.Version // use full version as commit hash if can't parse - } - - // Format the version based on module's major version from path - targetVersion := fmt.Sprintf("%s.0.0-%s-%s", majorVersion, timestamp, commitHash) - if majorVersion == "v0" { - targetVersion = fmt.Sprintf("v0.0.0-%s-%s", timestamp, commitHash) - } - - // Find current version + // Get major version from module path suffix (/v2, /v3, etc) + majorVersion := "v0" + if idx := strings.LastIndex(modulePath, "/v"); idx != -1 { + versionSuffix := modulePath[idx+1:] // get everything after the /v + if _, err := fmt.Sscanf(versionSuffix, "v%d", new(int)); err == nil { + majorVersion = versionSuffix + } + } + + // Create proper pseudo-version using x/mod/module + // Note: older version parameter is empty since we're creating a new pseudo-version + pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:12]) + + // Find and update version for _, req := range f.Require { if req.Mod.Path == modulePath { moduleExists = true if u.config.DryRun { - log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, targetVersion) + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, pseudoVersion) continue } - if err := f.AddRequire(modulePath, targetVersion); err != nil { + if err := f.AddRequire(modulePath, pseudoVersion); err != nil { return fmt.Errorf("failed to add requirement: %w", err) } break diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index 5a070f1e768..a4842057d5e 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -83,6 +83,9 @@ func (m *mockSystemOperator) WriteFile(path string, data []byte, perm uint32) er } func TestUpdater_Run(t *testing.T) { + testTime := time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC) + testSHA := "ac7a7395feed" + tests := []struct { name string config *Config @@ -157,12 +160,12 @@ require github.com/example/mod v1.0.0 name: "updates v2 module with timestamp", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/smartcontractkit/chainlink/v2", - Version: "v2.0.0-20241122182110-ac7a7395feed", - }, + sha: "ac7a7395feed", // Set exact SHA + updateTime: time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC), // Set exact time }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -181,12 +184,12 @@ require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395f name: "updates v0 module with timestamp", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/deployment"}, + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/smartcontractkit/chainlink/deployment", - Version: "v2.0.0-20241122182110-ac7a7395feed", - }, + sha: testSHA, + updateTime: testTime, }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -207,13 +210,13 @@ require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-a ModulesToUpdate: []string{ "github.com/smartcontractkit/chainlink/v2", "github.com/smartcontractkit/chainlink/deployment", - }, + }, + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/smartcontractkit/chainlink/v2", - Version: "v2.0.0-20241122182110-ac7a7395feed", - }, + sha: testSHA, + updateTime: testTime, }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -238,12 +241,12 @@ require ( name: "updates v3 module with timestamp", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v3"}, + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/smartcontractkit/chainlink/v3", - Version: "v2.0.0-20241122182110-ac7a7395feed", // Version from main branch - }, + sha: testSHA, + updateTime: testTime, }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() From 9c22ee27b8849273d030b06817256155633c3e5b Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:21:14 -0500 Subject: [PATCH 17/47] Upate pseudo-versions for local replaces to latest time/sha --- core/scripts/go.mod | 4 ++-- deployment/go.mod | 2 +- integration-tests/go.mod | 4 ++-- integration-tests/load/go.mod | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 6bdf9205d8b..8c77fd7e2ca 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -10,8 +10,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b ) require ( diff --git a/deployment/go.mod b/deployment/go.mod index 5bc3a4e0c8a..8ce946fcc69 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -7,7 +7,7 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ // Using a separate inline `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b require ( github.com/Khan/genqlient v0.7.0 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 1359d23b63e..861d6d21fe8 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -10,8 +10,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b ) require ( diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index c578d2a3413..20edadfd6c1 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -12,9 +12,9 @@ replace github.com/smartcontractkit/chainlink/integration-tests => ../ // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed - github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241122182110-ac7a7395feed - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b ) require ( From dd2390c9b011e7c97183486181822c655786a996 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:21:41 -0500 Subject: [PATCH 18/47] Cleanup --- .../internal/updater/config.go | 6 +- .../internal/updater/interfaces.go | 24 ++--- .../internal/updater/module_operator.go | 78 +++++---------- .../internal/updater/system_operator.go | 16 +-- .../internal/updater/updater.go | 97 +++++++++---------- .../internal/updater/updater_test.go | 55 +++++++---- 6 files changed, 128 insertions(+), 148 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go index a77e20fe840..331195ebb90 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -10,8 +10,8 @@ type Config struct { BranchTrunk string DryRun bool ShowVersion bool - OrgName string // Set during runtime - RepoName string // Set during runtime + OrgName string + RepoName string } func ParseFlags(args []string, version string) (*Config, error) { @@ -25,6 +25,8 @@ func ParseFlags(args []string, version string) (*Config, error) { flags.StringVar(&cfg.BranchTrunk, "branch-trunk", "develop", "Branch to get SHA from") flags.BoolVar(&cfg.DryRun, "dry-run", false, "Preview changes without applying them") flags.BoolVar(&cfg.ShowVersion, "version", false, "Show version information") + flags.StringVar(&cfg.OrgName, "org-name", "smartcontractkit", "GitHub organization name") + flags.StringVar(&cfg.RepoName, "repo-name", "chainlink", "GitHub repository name") if err := flags.Parse(args); err != nil { return nil, err diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index 20c2cc02d9f..400d1bbb7c6 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -2,31 +2,27 @@ package updater import ( "fmt" + "os" "time" + "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) // Errors var ( - ErrInvalidConfig = fmt.Errorf("invalid configuration") - ErrModOperation = fmt.Errorf("module operation failed") - ErrFileOperation = fmt.Errorf("file operation failed") + ErrModOperation = fmt.Errorf("module operation failed") ) +// ModuleOperator handles module-related operations type ModuleOperator interface { - // GetLatestVersion gets the latest pseudo-version based on current git state + GetGitInfo(remote, branch string) (string, time.Time, error) GetLatestVersion(modulePath string) (module.Version, error) - // GetModuleInfo gets version info including timestamp - GetModuleInfo(modulePath string) (module.Version, time.Time, error) - // ParseModulePathParts extracts org and repo from module path - ParseModulePathParts(modulePath string) (org, repo string, err error) - // GetGitInfo gets the latest git SHA and commit time - GetGitInfo(remote, branch string) (sha string, commitTime time.Time, err error) + UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) } -type Updater struct { - mod ModuleOperator - system SystemOperator - config *Config +// SystemOperator handles file system operations +type SystemOperator interface { + ReadFile(path string) ([]byte, error) + WriteFile(path string, data []byte, perm os.FileMode) error } diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go index 41a09249f15..666d7e9fdd4 100644 --- a/tools/gomod-required-updater/internal/updater/module_operator.go +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -1,7 +1,6 @@ package updater import ( - "encoding/json" "fmt" "os/exec" "regexp" @@ -17,11 +16,11 @@ const ( ) // getMajorVersion extracts the major version number from a module path -// Returns "v2" for /v2, "v0" for no version suffix +// Returns "v2" for /v2, "v1" for no version suffix func getMajorVersion(modulePath string) string { re := regexp.MustCompile(majorVersionPattern) if match := re.FindString(modulePath); match != "" { - return "v" + strings.TrimPrefix(match, "/v") + return strings.TrimPrefix(match, "/") } return "v0" } @@ -80,63 +79,34 @@ func (m *moduleOperator) GetLatestVersion(modulePath string) (module.Version, er }, nil } -func (m *moduleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) error { - for _, req := range modFile.Require { - if strings.HasPrefix(req.Mod.Path, "github.com/smartcontractkit/chainlink") { - // Check if this is a v2+ module path - modVersion := getMajorVersion(req.Mod.Path) - if modVersion != "v0" { - // If module path contains v2+, maintain that version - newVer := strings.Replace(newVersion, "v0", modVersion, 1) - req.Mod.Version = newVer - } else { - // For modules without version in path, use newVersion as-is - req.Mod.Version = newVersion - } - } +// extractVersionParts splits a pseudo-version into its components +func extractVersionParts(version string) (base, timestamp, sha string) { + parts := strings.Split(version, "-") + if len(parts) == 3 { + return parts[0], parts[1], parts[2] } - return nil -} - -type moduleInfo struct { - Path string `json:"Path"` - Version string `json:"Version"` - Time time.Time `json:"Time"` + return "", "", "" } -// GetModuleInfo gets version info including timestamp -func (m *moduleOperator) GetModuleInfo(modulePath string) (module.Version, time.Time, error) { - cmd := exec.Command("go", "list", "-m", "-json", modulePath+"@latest") - out, err := cmd.Output() - if err != nil { - return module.Version{}, time.Time{}, fmt.Errorf("%w: %v", ErrModOperation, err) - } - - var info moduleInfo - if err := json.Unmarshal(out, &info); err != nil { - return module.Version{}, time.Time{}, fmt.Errorf("%w: failed to decode response: %v", ErrModOperation, err) +// UpdateRequiredVersions identifies modules that need version updates +func (m *moduleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) { + _, _, sha := extractVersionParts(newVersion) + if sha == "" { + return nil, fmt.Errorf("%w: invalid version format: %s", ErrModOperation, newVersion) } - return module.Version{ - Path: info.Path, - Version: info.Version, - }, info.Time, nil -} - -// ParseModulePathParts extracts org and repo from module path -func (m *moduleOperator) ParseModulePathParts(modulePath string) (org, repo string, err error) { - parts := strings.Split(modulePath, "/") - if len(parts) < 3 { - return "", "", fmt.Errorf("%w: invalid module path format: %s", ErrModOperation, modulePath) + var modulesToUpdate []string + localModules := make(map[string]bool) + for _, rep := range modFile.Replace { + if rep.New.Version == "" { + localModules[rep.Old.Path] = true + } } - org = parts[1] - repo = parts[2] - - // Strip version suffix if present (e.g., "repo/v2" -> "repo") - if i := strings.Index(repo, "/v"); i != -1 { - repo = repo[:i] + for _, req := range modFile.Require { + if localModules[req.Mod.Path] { + modulesToUpdate = append(modulesToUpdate, req.Mod.Path) + } } - - return org, repo, nil + return modulesToUpdate, nil } diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index 7965a1c0297..3877b90a89f 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -1,18 +1,12 @@ package updater import ( - "io" "os" ) -type SystemOperator interface { - ReadFile(path string) ([]byte, error) - WriteFile(filename string, data []byte, perm uint32) error -} - type systemOperator struct { - stdout io.Writer - stderr io.Writer + stdout *os.File + stderr *os.File } func NewSystemOperator() SystemOperator { @@ -22,10 +16,10 @@ func NewSystemOperator() SystemOperator { } } -func (so *systemOperator) ReadFile(path string) ([]byte, error) { +func (s *systemOperator) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } -func (so *systemOperator) WriteFile(filename string, data []byte, perm uint32) error { - return os.WriteFile(filename, data, os.FileMode(perm)) +func (s *systemOperator) WriteFile(path string, data []byte, perm os.FileMode) error { + return os.WriteFile(path, data, perm) } diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 3a45ec65860..450c8ff31b3 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -10,6 +10,17 @@ import ( "golang.org/x/mod/module" ) +const ( + goModFile = "go.mod" +) + +type Updater struct { + mod ModuleOperator + system SystemOperator + config *Config +} + + // New creates a new Updater func New(mod ModuleOperator, system SystemOperator, config *Config) *Updater { return &Updater{ @@ -27,24 +38,11 @@ func (u *Updater) Run() error { log.Printf("Starting update process for modules: %v", u.config.ModulesToUpdate) } - // Get org and repo info from current module first - content, err := u.system.ReadFile("go.mod") + // Use helper method instead of direct file read + f, err := u.readModFile() if err != nil { - return fmt.Errorf("failed to read go.mod: %w", err) - } - - f, err := modfile.Parse("go.mod", content, nil) - if err != nil { - return fmt.Errorf("failed to parse go.mod: %w", err) - } - - // Get org/repo from the current module path - org, repo, err := u.mod.ParseModulePathParts(f.Module.Mod.Path) - if err != nil { - return fmt.Errorf("failed to get repo info from current module: %w", err) + return err } - u.config.OrgName = org - u.config.RepoName = repo // Find modules to update first if none specified if len(u.config.ModulesToUpdate) == 0 { @@ -75,30 +73,15 @@ func (u *Updater) Run() error { // updateGoMod updates the go.mod file with new pseudo-versions func (u *Updater) updateGoMod(path string, sha string, commitTime time.Time) error { - content, err := u.system.ReadFile(path) - if err != nil { - return err - } - - f, err := modfile.Parse(path, content, nil) + // Use helper method instead of direct file read + f, err := u.readModFile() if err != nil { return err } for _, modulePath := range u.config.ModulesToUpdate { moduleExists := false - - // Get major version from module path suffix (/v2, /v3, etc) - majorVersion := "v0" - if idx := strings.LastIndex(modulePath, "/v"); idx != -1 { - versionSuffix := modulePath[idx+1:] // get everything after the /v - if _, err := fmt.Sscanf(versionSuffix, "v%d", new(int)); err == nil { - majorVersion = versionSuffix - } - } - - // Create proper pseudo-version using x/mod/module - // Note: older version parameter is empty since we're creating a new pseudo-version + majorVersion := getMajorVersion(modulePath) pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:12]) // Find and update version @@ -122,16 +105,7 @@ func (u *Updater) updateGoMod(path string, sha string, commitTime time.Time) err } } - newContent, err := f.Format() - if err != nil { - return fmt.Errorf("failed to format go.mod: %w", err) - } - - if err := u.system.WriteFile(path, newContent, 0644); err != nil { - return fmt.Errorf("failed to write go.mod: %w", err) - } - - return nil + return u.writeModFile(f) } // findLocalReplaceModules finds modules with local replace directives @@ -140,12 +114,8 @@ func (u *Updater) findLocalReplaceModules() ([]string, error) { seen := make(map[string]bool) orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) - content, err := u.system.ReadFile("go.mod") - if err != nil { - return nil, err - } - - f, err := modfile.Parse("go.mod", content, nil) + // Use helper method instead of direct file read + f, err := u.readModFile() if err != nil { return nil, err } @@ -170,3 +140,30 @@ func isLocalPath(path string) bool { strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../") } + +func (u *Updater) readModFile() (*modfile.File, error) { + content, err := u.system.ReadFile("go.mod") + if err != nil { + return nil, fmt.Errorf("failed to read go.mod: %w", err) + } + + f, err := modfile.Parse("go.mod", content, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go.mod: %w", err) + } + + return f, nil +} + +func (u *Updater) writeModFile(f *modfile.File) error { + content, err := f.Format() + if err != nil { + return fmt.Errorf("failed to format go.mod: %w", err) + } + + if err := u.system.WriteFile("go.mod", content, 0644); err != nil { + return fmt.Errorf("failed to write go.mod: %w", err) + } + + return nil +} diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index a4842057d5e..6e1ed6c16f8 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -2,20 +2,23 @@ package updater import ( "fmt" + "os" "testing" "time" + "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) // Mock implementations type mockModuleOperator struct { - version module.Version - updateTime time.Time - org string - repo string - sha string - err error + version module.Version + updateTime time.Time + org string + repo string + sha string + err error + modulesToUpdate []string // Add this new field } func (m *mockModuleOperator) GetLatestVersion(modulePath string) (module.Version, error) { @@ -52,6 +55,24 @@ func (m *mockModuleOperator) GetGitInfo(remote, branch string) (string, time.Tim return m.sha, m.updateTime, nil } +// Add the missing method +func (m *mockModuleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) { + if m.err != nil { + return nil, m.err + } + if m.modulesToUpdate != nil { + return m.modulesToUpdate, nil + } + // Default behavior: return the modules that have local replace directives + var modules []string + for _, rep := range modFile.Replace { + if rep.New.Version == "" { // Local replace has empty version + modules = append(modules, rep.Old.Path) + } + } + return modules, nil +} + type mockSystemOperator struct { files map[string][]byte err error @@ -74,7 +95,7 @@ func (m *mockSystemOperator) ReadFile(path string) ([]byte, error) { return content, nil } -func (m *mockSystemOperator) WriteFile(path string, data []byte, perm uint32) error { +func (m *mockSystemOperator) WriteFile(path string, data []byte, perm os.FileMode) error { if m.err != nil { return m.err } @@ -160,11 +181,11 @@ require github.com/example/mod v1.0.0 name: "updates v2 module with timestamp", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ - sha: "ac7a7395feed", // Set exact SHA + sha: "ac7a7395feed", // Set exact SHA updateTime: time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC), // Set exact time }, sysOp: func() *mockSystemOperator { @@ -184,8 +205,8 @@ require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395f name: "updates v0 module with timestamp", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/deployment"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ sha: testSHA, @@ -210,9 +231,9 @@ require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-a ModulesToUpdate: []string{ "github.com/smartcontractkit/chainlink/v2", "github.com/smartcontractkit/chainlink/deployment", - }, - RepoRemote: "origin", - BranchTrunk: "develop", + }, + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ sha: testSHA, @@ -241,8 +262,8 @@ require ( name: "updates v3 module with timestamp", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v3"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", }, modOp: &mockModuleOperator{ sha: testSHA, From b9dc91ac9e21f06aad2dc6b1f41fa62e8604f040 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:23:36 -0500 Subject: [PATCH 19/47] Update bin path --- .github/workflows/ci-gomod-required-updater.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml index 5108ebbbb25..2a2e7773b40 100644 --- a/.github/workflows/ci-gomod-required-updater.yml +++ b/.github/workflows/ci-gomod-required-updater.yml @@ -12,4 +12,6 @@ jobs: - name: Setup Go uses: ./.github/actions/setup-go - name: Build binary - run: go build ./tools/gomod-required-updater/cmd/gomod-required-updater + run: | + go build -o dist/gomod-required-updater \ + ./tools/gomod-required-updater/cmd/gomod-required-updater From f8fbc9b3af3ffb68edd1fdc2c892bf8f10e3141c Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:28:00 -0500 Subject: [PATCH 20/47] Fix README --- tools/gomod-required-updater/README.md | 54 +++----------------------- 1 file changed, 6 insertions(+), 48 deletions(-) diff --git a/tools/gomod-required-updater/README.md b/tools/gomod-required-updater/README.md index 24679131966..4e7fddc5f8c 100644 --- a/tools/gomod-required-updater/README.md +++ b/tools/gomod-required-updater/README.md @@ -2,72 +2,30 @@ Updates required module versions in go.mod files to match the latest git SHA from a remote branch. -## Features - -- Auto-detect and update modules with local replace directives -- Preview changes with dry run mode - ## Configuration -Optional TOML config. - -```toml -# List of modules to update -modules = [ - "github.com/smartcontractkit/chainlink/v2" -] -``` - Command Line Flags: ```shell Optional: + -org-name Organization name to update modules for (default: smartcontractkit) + -repo-name Repository name to update modules for (default: chainlink) -repo-remote Git remote to use (default: origin) -branch-trunk Branch to get SHA from (default: develop) -dry-run Preview changes without applying them (default: false) + - ``` ## Installation -The installed binary will be placed in your `$GOPATH/bin` directory. Make sure this directory is in your system's PATH to run the command from anywhere. +The installed binary will be placed in your `$GOPATH/bin` directory. Make sure this directory is in your system's PATH to run the command from anywhere. From the root of this repository, run: ```shell -go install github.com/smartcontractkit/chainlink/tools/gomod-required-updater +go install ./tools/gomod-required-updater/cmd/gomod-required-updater ``` ## Usage Examples -Update Specific Modules: - -```shell -# Update single module -gomod-required-updater -module github.com/org/repo` - -# Update multiple modules -gomod-required-updater -module github.com/org/repo1 -module github.com/org/repo2 - -# Update using config file -gomod-required-updater -config modules.toml - -# Using different remote/branch -gomod-required-updater -module github.com/org/repo -repo-remote upstream -branch-trunk main -``` - -Auto-detect and Update Local Modules: - ```shell -# Update all local modules that have replace directives -gomod-required-updater -update-org-modules - -# Preview changes first -gomod-required-updater -update-org-modules -dry-run +make gomodrequiredupdater ``` - -## Notes - -- When using multiple module sources, precedence is: - 1. Command line `-module` flags - 2. Config file modules - 3. Auto-detected modules via `-update-org-modules` flag -- Use the `-dry-run` flag to safely preview changes -- Local replace directives are preserved during updates From bf12443e84f0088001a336b9dfb59c5e16c958c4 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Sat, 23 Nov 2024 18:12:12 -0500 Subject: [PATCH 21/47] Allow custom org/repo name and apply better validation to git cmds --- .../internal/updater/config.go | 33 +++- .../internal/updater/config_test.go | 50 +++++- .../internal/updater/interfaces.go | 34 ++-- .../internal/updater/module_operator.go | 47 +++++- .../internal/updater/system_operator.go | 25 +-- .../internal/updater/updater.go | 149 +++++++++--------- 6 files changed, 231 insertions(+), 107 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-required-updater/internal/updater/config.go index 331195ebb90..ee14e9488a5 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-required-updater/internal/updater/config.go @@ -2,6 +2,14 @@ package updater import ( "flag" + "fmt" +) + +const ( + DefaultRepoRemote = "origin" + DefaultBranchTrunk = "develop" + DefaultOrgName = "smartcontractkit" + DefaultRepoName = "chainlink" ) type Config struct { @@ -21,12 +29,12 @@ func ParseFlags(args []string, version string) (*Config, error) { ModulesToUpdate: make([]string, 0), } - flags.StringVar(&cfg.RepoRemote, "repo-remote", "origin", "Git remote to use") - flags.StringVar(&cfg.BranchTrunk, "branch-trunk", "develop", "Branch to get SHA from") + flags.StringVar(&cfg.RepoRemote, "repo-remote", DefaultRepoRemote, "Git remote to use") + flags.StringVar(&cfg.BranchTrunk, "branch-trunk", DefaultBranchTrunk, "Branch to get SHA from") flags.BoolVar(&cfg.DryRun, "dry-run", false, "Preview changes without applying them") flags.BoolVar(&cfg.ShowVersion, "version", false, "Show version information") - flags.StringVar(&cfg.OrgName, "org-name", "smartcontractkit", "GitHub organization name") - flags.StringVar(&cfg.RepoName, "repo-name", "chainlink", "GitHub repository name") + flags.StringVar(&cfg.OrgName, "org-name", DefaultOrgName, "GitHub organization name") + flags.StringVar(&cfg.RepoName, "repo-name", DefaultRepoName, "GitHub repository name") if err := flags.Parse(args); err != nil { return nil, err @@ -39,5 +47,22 @@ func (c *Config) Validate() error { if c.ShowVersion { return nil } + + if c.RepoRemote == "" { + return fmt.Errorf("%w: repo-remote is required", ErrInvalidConfig) + } + + if c.BranchTrunk == "" { + return fmt.Errorf("%w: branch-trunk is required", ErrInvalidConfig) + } + + if c.OrgName == "" { + return fmt.Errorf("%w: org-name is required", ErrInvalidConfig) + } + + if c.RepoName == "" { + return fmt.Errorf("%w: repo-name is required", ErrInvalidConfig) + } + return nil } diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-required-updater/internal/updater/config_test.go index b46ea0a9c18..79fe72a4b9b 100644 --- a/tools/gomod-required-updater/internal/updater/config_test.go +++ b/tools/gomod-required-updater/internal/updater/config_test.go @@ -5,29 +5,67 @@ import "testing" func TestConfig_Validate(t *testing.T) { tests := []struct { name string - cfg *Config + config *Config wantErr bool }{ { name: "valid config", - cfg: &Config{ - RepoRemote: "origin", - BranchTrunk: "main", + config: &Config{ + RepoRemote: DefaultRepoRemote, + BranchTrunk: DefaultBranchTrunk, + OrgName: DefaultOrgName, + RepoName: DefaultRepoName, }, wantErr: false, }, { name: "version flag bypasses validation", - cfg: &Config{ + config: &Config{ ShowVersion: true, }, wantErr: false, }, + { + name: "missing repo remote", + config: &Config{ + BranchTrunk: DefaultBranchTrunk, + OrgName: DefaultOrgName, + RepoName: DefaultRepoName, + }, + wantErr: true, + }, + { + name: "missing branch trunk", + config: &Config{ + RepoRemote: DefaultRepoRemote, + OrgName: DefaultOrgName, + RepoName: DefaultRepoName, + }, + wantErr: true, + }, + { + name: "missing org name", + config: &Config{ + RepoRemote: DefaultRepoRemote, + BranchTrunk: DefaultBranchTrunk, + RepoName: DefaultRepoName, + }, + wantErr: true, + }, + { + name: "missing repo name", + config: &Config{ + RepoRemote: DefaultRepoRemote, + BranchTrunk: DefaultBranchTrunk, + OrgName: DefaultOrgName, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := tt.cfg.Validate() + err := tt.config.Validate() if (err != nil) != tt.wantErr { t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index 400d1bbb7c6..b8e82ce8e97 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -1,3 +1,6 @@ +// Package updater provides functionality to update Go module versions in go.mod files. +// It specializes in handling local replace directives and maintaining consistent +// versioning across related modules. package updater import ( @@ -9,20 +12,33 @@ import ( "golang.org/x/mod/module" ) -// Errors +const ( + gitSHALength = 12 +) + +// Errors that can be returned by the updater package var ( - ErrModOperation = fmt.Errorf("module operation failed") + // ErrModOperation indicates a failure in a module-related operation + ErrModOperation = fmt.Errorf("module operation failed") + // ErrInvalidConfig indicates invalid configuration parameters + ErrInvalidConfig = fmt.Errorf("invalid configuration") ) -// ModuleOperator handles module-related operations +// ModuleOperator handles Git repository operations and module version management. +// It provides functionality to retrieve Git information and manage module versions. type ModuleOperator interface { - GetGitInfo(remote, branch string) (string, time.Time, error) - GetLatestVersion(modulePath string) (module.Version, error) - UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) + // GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository + GetGitInfo(remote, branch string) (string, time.Time, error) + // GetLatestVersion constructs a pseudo-version for a module based on Git information + GetLatestVersion(modulePath string) (module.Version, error) + // UpdateRequiredVersions identifies modules that need version updates + UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) } -// SystemOperator handles file system operations +// SystemOperator provides an interface for file system operations. type SystemOperator interface { - ReadFile(path string) ([]byte, error) - WriteFile(path string, data []byte, perm os.FileMode) error + // ReadFile reads the entire contents of a file + ReadFile(path string) ([]byte, error) + // WriteFile writes data to a file with specific permissions + WriteFile(path string, data []byte, perm os.FileMode) error } diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go index 666d7e9fdd4..350f8375ae6 100644 --- a/tools/gomod-required-updater/internal/updater/module_operator.go +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -1,6 +1,7 @@ package updater import ( + "context" "fmt" "os/exec" "regexp" @@ -13,6 +14,10 @@ import ( const ( majorVersionPattern = `/v\d+$` + gitTimeout = 30 * time.Second + gitTimeFormat = time.RFC3339 + gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` + gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` ) // getMajorVersion extracts the major version number from a module path @@ -29,6 +34,7 @@ type moduleOperator struct { config *Config } +// NewModuleOperator creates a new ModuleOperator func NewModuleOperator(config *Config) ModuleOperator { if config.RepoRemote == "" { config.RepoRemote = "origin" @@ -41,29 +47,62 @@ func NewModuleOperator(config *Config) ModuleOperator { } } +// validateGitInput checks if the remote and branch are in the correct format +func validateGitInput(remote, branch string) error { + remoteRE := regexp.MustCompile(gitRemotePattern) + if !remoteRE.MatchString(remote) { + return fmt.Errorf("%w: invalid git remote format", ErrModOperation) + } + + branchRE := regexp.MustCompile(gitBranchPattern) + if !branchRE.MatchString(branch) { + return fmt.Errorf("%w: invalid git branch format", ErrModOperation) + } + return nil +} + +// GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, error) { + if err := validateGitInput(remote, branch); err != nil { + return "", time.Time{}, err + } + + ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) + defer cancel() + // Get latest SHA - cmd := exec.Command("git", "ls-remote", remote, "refs/heads/"+branch) + cmd := exec.CommandContext(ctx, "git", "ls-remote", remote, "refs/heads/"+branch) out, err := cmd.Output() if err != nil { return "", time.Time{}, fmt.Errorf("%w: failed to get SHA: %v", ErrModOperation, err) } + if len(out) == 0 { + return "", time.Time{}, fmt.Errorf("%w: no output from git ls-remote", ErrModOperation) + } sha := strings.Split(string(out), "\t")[0] + if len(sha) == 0 { + return "", time.Time{}, fmt.Errorf("%w: empty SHA from git ls-remote", ErrModOperation) + } // Get commit timestamp - cmd = exec.Command("git", "show", "-s", "--format=%cI", sha) + cmd = exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) out, err = cmd.Output() if err != nil { return "", time.Time{}, fmt.Errorf("%w: failed to get commit time: %v", ErrModOperation, err) } - commitTime, err := time.Parse(time.RFC3339, strings.TrimSpace(string(out))) + if len(out) == 0 { + return "", time.Time{}, fmt.Errorf("%w: no output from git show", ErrModOperation) + } + + commitTime, err := time.Parse(gitTimeFormat, strings.TrimSpace(string(out))) if err != nil { return "", time.Time{}, fmt.Errorf("%w: failed to parse commit time: %v", ErrModOperation, err) } - return sha[:12], commitTime, nil + return sha[:gitSHALength], commitTime, nil } +// GetLatestVersion retrieves the latest pseudo-version for a module func (m *moduleOperator) GetLatestVersion(modulePath string) (module.Version, error) { sha, commitTime, err := m.GetGitInfo(m.config.RepoRemote, m.config.BranchTrunk) if err != nil { diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index 3877b90a89f..7920ab234ce 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -2,24 +2,29 @@ package updater import ( "os" + "path/filepath" ) -type systemOperator struct { - stdout *os.File - stderr *os.File -} +const ( + defaultFileMode = 0644 +) + +type systemOperator struct{} func NewSystemOperator() SystemOperator { - return &systemOperator{ - stdout: os.Stdout, - stderr: os.Stderr, - } + return &systemOperator{} } func (s *systemOperator) ReadFile(path string) ([]byte, error) { - return os.ReadFile(path) + path = filepath.Clean(path) + return os.ReadFile(path) } func (s *systemOperator) WriteFile(path string, data []byte, perm os.FileMode) error { - return os.WriteFile(path, data, perm) + path = filepath.Clean(path) + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + return os.WriteFile(path, data, perm) } diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 450c8ff31b3..0f79aedba89 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -3,6 +3,7 @@ package updater import ( "fmt" "log" + "os" "strings" "time" @@ -11,7 +12,8 @@ import ( ) const ( - goModFile = "go.mod" + goModFile = "go.mod" + goModFileMode = 0644 ) type Updater struct { @@ -30,82 +32,79 @@ func New(mod ModuleOperator, system SystemOperator, config *Config) *Updater { } } -// Run the update process +// Run starts the module update process func (u *Updater) Run() error { - if len(u.config.ModulesToUpdate) == 0 { - log.Printf("No modules specified, will auto-detect modules with local replace directives") - } else { - log.Printf("Starting update process for modules: %v", u.config.ModulesToUpdate) - } + logger := log.New(os.Stdout, "", log.LstdFlags) - // Use helper method instead of direct file read - f, err := u.readModFile() - if err != nil { - return err - } + if len(u.config.ModulesToUpdate) == 0 { + logger.Printf("info: auto-detecting modules with local replace directives") + } else { + logger.Printf("info: updating modules: %v", u.config.ModulesToUpdate) + } - // Find modules to update first if none specified - if len(u.config.ModulesToUpdate) == 0 { - modulesToAdd, err := u.findLocalReplaceModules() - if err != nil { - return fmt.Errorf("failed to find local replace modules: %w", err) - } - if len(modulesToAdd) == 0 { - log.Printf("No modules found to update in %s", f.Module.Mod.Path) - return nil // This is now a non-error case - } - u.config.ModulesToUpdate = modulesToAdd - log.Printf("Found %d modules with local replace directives: %v", len(modulesToAdd), modulesToAdd) - } + f, err := u.readModFile() + if err != nil { + return err + } - // Get commit info once for all modules - sha, commitTime, err := u.mod.GetGitInfo(u.config.RepoRemote, u.config.BranchTrunk) - if err != nil { - return fmt.Errorf("failed to get git info: %w", err) - } + // Find modules to update first if none specified + if len(u.config.ModulesToUpdate) == 0 { + modulesToAdd, err := u.findLocalReplaceModules() + if err != nil { + return fmt.Errorf("failed to find local replace modules: %w", err) + } + if len(modulesToAdd) == 0 { + logger.Printf("info: no modules found to update in %s", f.Module.Mod.Path) + return nil // This is now a non-error case + } + u.config.ModulesToUpdate = modulesToAdd + logger.Printf("info: found %d modules with local replace directives: %v", len(modulesToAdd), modulesToAdd) + } - if err := u.updateGoMod("go.mod", sha, commitTime); err != nil { - return fmt.Errorf("error updating go.mod: %w", err) - } + // Get commit info once for all modules + sha, commitTime, err := u.mod.GetGitInfo(u.config.RepoRemote, u.config.BranchTrunk) + if err != nil { + return fmt.Errorf("failed to get git info: %w", err) + } + + // Update the modules in the same file handle + if err := u.updateGoMod(f, sha, commitTime); err != nil { + return fmt.Errorf("error updating %s: %w", goModFile, err) + } - return nil + // Write the changes + return u.writeModFile(f) } // updateGoMod updates the go.mod file with new pseudo-versions -func (u *Updater) updateGoMod(path string, sha string, commitTime time.Time) error { - // Use helper method instead of direct file read - f, err := u.readModFile() - if err != nil { - return err - } - - for _, modulePath := range u.config.ModulesToUpdate { - moduleExists := false - majorVersion := getMajorVersion(modulePath) - pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:12]) - - // Find and update version - for _, req := range f.Require { - if req.Mod.Path == modulePath { - moduleExists = true - if u.config.DryRun { - log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, pseudoVersion) - continue - } - - if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("failed to add requirement: %w", err) - } - break - } - } - - if !moduleExists { - continue - } - } +func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) error { + for _, modulePath := range u.config.ModulesToUpdate { + moduleExists := false + majorVersion := getMajorVersion(modulePath) + pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:gitSHALength]) + + // Find and update version + for _, req := range f.Require { + if req.Mod.Path == modulePath { + moduleExists = true + if u.config.DryRun { + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, pseudoVersion) + continue + } + + if err := f.AddRequire(modulePath, pseudoVersion); err != nil { + return fmt.Errorf("failed to add requirement: %w", err) + } + break + } + } + + if !moduleExists { + continue + } + } - return u.writeModFile(f) + return nil } // findLocalReplaceModules finds modules with local replace directives @@ -141,28 +140,30 @@ func isLocalPath(path string) bool { strings.HasPrefix(path, "../") } +// readModFile reads the go.mod file func (u *Updater) readModFile() (*modfile.File, error) { - content, err := u.system.ReadFile("go.mod") + content, err := u.system.ReadFile(goModFile) if err != nil { - return nil, fmt.Errorf("failed to read go.mod: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", goModFile, err) } - f, err := modfile.Parse("go.mod", content, nil) + f, err := modfile.Parse(goModFile, content, nil) if err != nil { - return nil, fmt.Errorf("failed to parse go.mod: %w", err) + return nil, fmt.Errorf("failed to parse %s: %w", goModFile, err) } return f, nil } +// writeModFile writes the go.mod file func (u *Updater) writeModFile(f *modfile.File) error { content, err := f.Format() if err != nil { - return fmt.Errorf("failed to format go.mod: %w", err) + return fmt.Errorf("failed to format %s: %w", goModFile, err) } - if err := u.system.WriteFile("go.mod", content, 0644); err != nil { - return fmt.Errorf("failed to write go.mod: %w", err) + if err := u.system.WriteFile(goModFile, content, goModFileMode); err != nil { + return fmt.Errorf("failed to write %s: %w", goModFile, err) } return nil From da8130bd4d037eb46eb1a554815288fb901817fc Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:05:25 -0500 Subject: [PATCH 22/47] Address linting --- .../internal/updater/interfaces.go | 30 +-- .../internal/updater/module_operator.go | 12 +- .../internal/updater/system_operator.go | 20 +- .../internal/updater/updater.go | 173 +++++++++--------- .../internal/updater/updater_test.go | 6 +- 5 files changed, 120 insertions(+), 121 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index b8e82ce8e97..6542bac611e 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -13,32 +13,32 @@ import ( ) const ( - gitSHALength = 12 + gitSHALength = 12 ) // Errors that can be returned by the updater package var ( - // ErrModOperation indicates a failure in a module-related operation - ErrModOperation = fmt.Errorf("module operation failed") - // ErrInvalidConfig indicates invalid configuration parameters - ErrInvalidConfig = fmt.Errorf("invalid configuration") + // ErrModOperation indicates a failure in a module-related operation + ErrModOperation = fmt.Errorf("module operation failed") + // ErrInvalidConfig indicates invalid configuration parameters + ErrInvalidConfig = fmt.Errorf("invalid configuration") ) // ModuleOperator handles Git repository operations and module version management. // It provides functionality to retrieve Git information and manage module versions. type ModuleOperator interface { - // GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository - GetGitInfo(remote, branch string) (string, time.Time, error) - // GetLatestVersion constructs a pseudo-version for a module based on Git information - GetLatestVersion(modulePath string) (module.Version, error) - // UpdateRequiredVersions identifies modules that need version updates - UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) + // GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository + GetGitInfo(remote, branch string) (string, time.Time, error) + // GetLatestVersion constructs a pseudo-version for a module based on Git information + GetLatestVersion(modulePath string) (module.Version, error) + // UpdateRequiredVersions identifies modules that need version updates + UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) } // SystemOperator provides an interface for file system operations. type SystemOperator interface { - // ReadFile reads the entire contents of a file - ReadFile(path string) ([]byte, error) - // WriteFile writes data to a file with specific permissions - WriteFile(path string, data []byte, perm os.FileMode) error + // ReadFile reads the entire contents of a file + ReadFile(path string) ([]byte, error) + // WriteFile writes data to a file with specific permissions + WriteFile(path string, data []byte, perm os.FileMode) error } diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go index 350f8375ae6..83039aa5a72 100644 --- a/tools/gomod-required-updater/internal/updater/module_operator.go +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -14,10 +14,10 @@ import ( const ( majorVersionPattern = `/v\d+$` - gitTimeout = 30 * time.Second - gitTimeFormat = time.RFC3339 - gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` - gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` + gitTimeout = 30 * time.Second + gitTimeFormat = time.RFC3339 + gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` + gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` ) // getMajorVersion extracts the major version number from a module path @@ -88,7 +88,7 @@ func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, e cmd = exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) out, err = cmd.Output() if err != nil { - return "", time.Time{}, fmt.Errorf("%w: failed to get commit time: %v", ErrModOperation, err) + return "", time.Time{}, fmt.Errorf("failed to get commit time: %w", err) } if len(out) == 0 { return "", time.Time{}, fmt.Errorf("%w: no output from git show", ErrModOperation) @@ -96,7 +96,7 @@ func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, e commitTime, err := time.Parse(gitTimeFormat, strings.TrimSpace(string(out))) if err != nil { - return "", time.Time{}, fmt.Errorf("%w: failed to parse commit time: %v", ErrModOperation, err) + return "", time.Time{}, fmt.Errorf("failed to parse commit time: %w", err) } return sha[:gitSHALength], commitTime, nil diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index 7920ab234ce..29e50f471aa 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -6,25 +6,25 @@ import ( ) const ( - defaultFileMode = 0644 + defaultFileMode = 0644 ) type systemOperator struct{} func NewSystemOperator() SystemOperator { - return &systemOperator{} + return &systemOperator{} } func (s *systemOperator) ReadFile(path string) ([]byte, error) { - path = filepath.Clean(path) - return os.ReadFile(path) + path = filepath.Clean(path) + return os.ReadFile(path) } func (s *systemOperator) WriteFile(path string, data []byte, perm os.FileMode) error { - path = filepath.Clean(path) - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - return os.WriteFile(path, data, perm) + path = filepath.Clean(path) + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + return os.WriteFile(path, data, perm) } diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-required-updater/internal/updater/updater.go index 0f79aedba89..b784765040a 100644 --- a/tools/gomod-required-updater/internal/updater/updater.go +++ b/tools/gomod-required-updater/internal/updater/updater.go @@ -12,8 +12,8 @@ import ( ) const ( - goModFile = "go.mod" - goModFileMode = 0644 + goModFile = "go.mod" + goModFileMode = 0644 ) type Updater struct { @@ -22,7 +22,6 @@ type Updater struct { config *Config } - // New creates a new Updater func New(mod ModuleOperator, system SystemOperator, config *Config) *Updater { return &Updater{ @@ -34,77 +33,77 @@ func New(mod ModuleOperator, system SystemOperator, config *Config) *Updater { // Run starts the module update process func (u *Updater) Run() error { - logger := log.New(os.Stdout, "", log.LstdFlags) - - if len(u.config.ModulesToUpdate) == 0 { - logger.Printf("info: auto-detecting modules with local replace directives") - } else { - logger.Printf("info: updating modules: %v", u.config.ModulesToUpdate) - } - - f, err := u.readModFile() - if err != nil { - return err - } - - // Find modules to update first if none specified - if len(u.config.ModulesToUpdate) == 0 { - modulesToAdd, err := u.findLocalReplaceModules() - if err != nil { - return fmt.Errorf("failed to find local replace modules: %w", err) - } - if len(modulesToAdd) == 0 { - logger.Printf("info: no modules found to update in %s", f.Module.Mod.Path) - return nil // This is now a non-error case - } - u.config.ModulesToUpdate = modulesToAdd - logger.Printf("info: found %d modules with local replace directives: %v", len(modulesToAdd), modulesToAdd) - } - - // Get commit info once for all modules - sha, commitTime, err := u.mod.GetGitInfo(u.config.RepoRemote, u.config.BranchTrunk) - if err != nil { - return fmt.Errorf("failed to get git info: %w", err) - } - - // Update the modules in the same file handle - if err := u.updateGoMod(f, sha, commitTime); err != nil { - return fmt.Errorf("error updating %s: %w", goModFile, err) - } - - // Write the changes - return u.writeModFile(f) + logger := log.New(os.Stdout, "", log.LstdFlags) + + if len(u.config.ModulesToUpdate) == 0 { + logger.Printf("info: auto-detecting modules with local replace directives") + } else { + logger.Printf("info: updating modules: %v", u.config.ModulesToUpdate) + } + + f, err := u.readModFile() + if err != nil { + return err + } + + // Find modules to update first if none specified + if len(u.config.ModulesToUpdate) == 0 { + u.config.ModulesToUpdate, err = u.findLocalReplaceModules() + if err != nil { + return fmt.Errorf("failed to find local replace modules: %w", err) + } + if len(u.config.ModulesToUpdate) == 0 { + logger.Printf("info: no modules found to update in %s", f.Module.Mod.Path) + return nil + } + logger.Printf("info: found %d modules with local replace directives: %v", + len(u.config.ModulesToUpdate), u.config.ModulesToUpdate) + } + + // Get commit info once for all modules + sha, commitTime, err := u.mod.GetGitInfo(u.config.RepoRemote, u.config.BranchTrunk) + if err != nil { + return fmt.Errorf("failed to get git info: %w", err) + } + + // Update the modules in the same file handle + if err := u.updateGoMod(f, sha, commitTime); err != nil { + return fmt.Errorf("error updating %s: %w", goModFile, err) + } + + // Write the changes + return u.writeModFile(f) } // updateGoMod updates the go.mod file with new pseudo-versions func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) error { - for _, modulePath := range u.config.ModulesToUpdate { - moduleExists := false - majorVersion := getMajorVersion(modulePath) - pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:gitSHALength]) - - // Find and update version - for _, req := range f.Require { - if req.Mod.Path == modulePath { - moduleExists = true - if u.config.DryRun { - log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, pseudoVersion) - continue - } - - if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("failed to add requirement: %w", err) - } - break - } - } - - if !moduleExists { - continue - } - } - - return nil + for _, modulePath := range u.config.ModulesToUpdate { + moduleExists := false + majorVersion := getMajorVersion(modulePath) + pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:gitSHALength]) + + // Find and update version + for _, req := range f.Require { + if req.Mod.Path == modulePath { + moduleExists = true + if u.config.DryRun { + log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, pseudoVersion) + continue + } + + if err := f.AddRequire(modulePath, pseudoVersion); err != nil { + return fmt.Errorf("failed to add requirement: %w", err) + } + break + } + } + + if !moduleExists { + continue + } + } + + return nil } // findLocalReplaceModules finds modules with local replace directives @@ -142,29 +141,29 @@ func isLocalPath(path string) bool { // readModFile reads the go.mod file func (u *Updater) readModFile() (*modfile.File, error) { - content, err := u.system.ReadFile(goModFile) - if err != nil { - return nil, fmt.Errorf("failed to read %s: %w", goModFile, err) - } + content, err := u.system.ReadFile(goModFile) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", goModFile, err) + } - f, err := modfile.Parse(goModFile, content, nil) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", goModFile, err) - } + f, err := modfile.Parse(goModFile, content, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", goModFile, err) + } - return f, nil + return f, nil } // writeModFile writes the go.mod file func (u *Updater) writeModFile(f *modfile.File) error { - content, err := f.Format() - if err != nil { - return fmt.Errorf("failed to format %s: %w", goModFile, err) - } + content, err := f.Format() + if err != nil { + return fmt.Errorf("failed to format %s: %w", goModFile, err) + } - if err := u.system.WriteFile(goModFile, content, goModFileMode); err != nil { - return fmt.Errorf("failed to write %s: %w", goModFile, err) - } + if err := u.system.WriteFile(goModFile, content, goModFileMode); err != nil { + return fmt.Errorf("failed to write %s: %w", goModFile, err) + } - return nil + return nil } diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-required-updater/internal/updater/updater_test.go index 6e1ed6c16f8..06591ea91a0 100644 --- a/tools/gomod-required-updater/internal/updater/updater_test.go +++ b/tools/gomod-required-updater/internal/updater/updater_test.go @@ -18,7 +18,7 @@ type mockModuleOperator struct { repo string sha string err error - modulesToUpdate []string // Add this new field + modulesToUpdate []string } func (m *mockModuleOperator) GetLatestVersion(modulePath string) (module.Version, error) { @@ -185,8 +185,8 @@ require github.com/example/mod v1.0.0 BranchTrunk: "develop", }, modOp: &mockModuleOperator{ - sha: "ac7a7395feed", // Set exact SHA - updateTime: time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC), // Set exact time + sha: "ac7a7395feed", + updateTime: time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC), }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() From c37c8f555e616ae06b3ccd4634e36c94298fcda7 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:21:05 -0500 Subject: [PATCH 23/47] Address linting 2 --- tools/gomod-required-updater/internal/updater/interfaces.go | 6 +++--- .../internal/updater/module_operator.go | 3 ++- .../internal/updater/system_operator.go | 4 ---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-required-updater/internal/updater/interfaces.go index 6542bac611e..f8dd222a038 100644 --- a/tools/gomod-required-updater/internal/updater/interfaces.go +++ b/tools/gomod-required-updater/internal/updater/interfaces.go @@ -4,7 +4,7 @@ package updater import ( - "fmt" + "errors" "os" "time" @@ -19,9 +19,9 @@ const ( // Errors that can be returned by the updater package var ( // ErrModOperation indicates a failure in a module-related operation - ErrModOperation = fmt.Errorf("module operation failed") + ErrModOperation = errors.New("module operation failed") // ErrInvalidConfig indicates invalid configuration parameters - ErrInvalidConfig = fmt.Errorf("invalid configuration") + ErrInvalidConfig = errors.New("invalid configuration") ) // ModuleOperator handles Git repository operations and module version management. diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go index 83039aa5a72..4f59b2f5a5b 100644 --- a/tools/gomod-required-updater/internal/updater/module_operator.go +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -63,6 +63,7 @@ func validateGitInput(remote, branch string) error { // GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, error) { + // Validate remote and branch against strict regex patterns before using in exec if err := validateGitInput(remote, branch); err != nil { return "", time.Time{}, err } @@ -74,7 +75,7 @@ func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, e cmd := exec.CommandContext(ctx, "git", "ls-remote", remote, "refs/heads/"+branch) out, err := cmd.Output() if err != nil { - return "", time.Time{}, fmt.Errorf("%w: failed to get SHA: %v", ErrModOperation, err) + return "", time.Time{}, fmt.Errorf("%w: failed to get SHA: %w", ErrModOperation, err) } if len(out) == 0 { return "", time.Time{}, fmt.Errorf("%w: no output from git ls-remote", ErrModOperation) diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-required-updater/internal/updater/system_operator.go index 29e50f471aa..49ed7ae4b40 100644 --- a/tools/gomod-required-updater/internal/updater/system_operator.go +++ b/tools/gomod-required-updater/internal/updater/system_operator.go @@ -5,10 +5,6 @@ import ( "path/filepath" ) -const ( - defaultFileMode = 0644 -) - type systemOperator struct{} func NewSystemOperator() SystemOperator { From 5f60e096bd143017dd2f8afdb5f0d3648bd645d1 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Sat, 23 Nov 2024 20:37:57 -0500 Subject: [PATCH 24/47] Ignore gosec lint for execs --- .../internal/updater/module_operator.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-required-updater/internal/updater/module_operator.go index 4f59b2f5a5b..ca57b5e8f8d 100644 --- a/tools/gomod-required-updater/internal/updater/module_operator.go +++ b/tools/gomod-required-updater/internal/updater/module_operator.go @@ -71,7 +71,10 @@ func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, e ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) defer cancel() - // Get latest SHA + // Safe to use remote/branch after validateGitInput ensures they match: + // - remote: ^[a-zA-Z0-9][-a-zA-Z0-9_.]*$ + // - branch: ^[a-zA-Z0-9][-a-zA-Z0-9/_]*$ + //nolint:gosec // Inputs are validated by regex patterns above cmd := exec.CommandContext(ctx, "git", "ls-remote", remote, "refs/heads/"+branch) out, err := cmd.Output() if err != nil { @@ -85,7 +88,7 @@ func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, e return "", time.Time{}, fmt.Errorf("%w: empty SHA from git ls-remote", ErrModOperation) } - // Get commit timestamp + //nolint:gosec // SHA is obtained from git ls-remote output above cmd = exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) out, err = cmd.Output() if err != nil { From f17e56d19218e4be2a71599b2587ab3e5bece19e Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:03:30 -0500 Subject: [PATCH 25/47] Update Make target name --- GNUmakefile | 4 ++-- tools/gomod-required-updater/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 373fe2bfb32..fd058a94aa3 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -144,8 +144,8 @@ presubmit: ## Format go files and imports. gomods: ## Install gomods go install github.com/jmank88/gomods@v0.1.4 -.PHONY: gomodrequiredupdater -gomodrequiredupdater: ## Run gomod-required-updater +.PHONY: gomodslocalupdate +gomodslocalupdate: ## Run gomod-required-updater go install ./tools/gomod-required-updater/cmd/gomod-required-updater gomods -w gomod-required-updater gomods tidy diff --git a/tools/gomod-required-updater/README.md b/tools/gomod-required-updater/README.md index 4e7fddc5f8c..6549555e22f 100644 --- a/tools/gomod-required-updater/README.md +++ b/tools/gomod-required-updater/README.md @@ -27,5 +27,5 @@ go install ./tools/gomod-required-updater/cmd/gomod-required-updater ## Usage Examples ```shell -make gomodrequiredupdater +make gomodslocalupdate ``` From b40b4f6cb018e8e148d1091f9ea9be5371da43a0 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:14:16 -0500 Subject: [PATCH 26/47] Avoid merge queue for build --- .github/workflows/ci-gomod-required-updater.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml index 2a2e7773b40..570e94d23e8 100644 --- a/.github/workflows/ci-gomod-required-updater.yml +++ b/.github/workflows/ci-gomod-required-updater.yml @@ -1,7 +1,6 @@ name: gomod-required-updater on: - merge_group: pull_request: jobs: From 6c6c8be76631828630d224992d09a6431a0880b6 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:52:45 -0500 Subject: [PATCH 27/47] Rename tool --- .../README.md | 4 ++-- .../cmd/gomod-local-update}/main.go | 13 ++++++++----- .../internal/updater/config.go | 2 +- .../internal/updater/config_test.go | 0 .../internal/updater/interfaces.go | 0 .../internal/updater/module_operator.go | 0 .../internal/updater/system_operator.go | 0 .../internal/updater/updater.go | 0 .../internal/updater/updater_test.go | 0 9 files changed, 11 insertions(+), 8 deletions(-) rename tools/{gomod-required-updater => gomod-local-update}/README.md (89%) rename tools/{gomod-required-updater/cmd/gomod-required-updater => gomod-local-update/cmd/gomod-local-update}/main.go (61%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/config.go (95%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/config_test.go (100%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/interfaces.go (100%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/module_operator.go (100%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/system_operator.go (100%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/updater.go (100%) rename tools/{gomod-required-updater => gomod-local-update}/internal/updater/updater_test.go (100%) diff --git a/tools/gomod-required-updater/README.md b/tools/gomod-local-update/README.md similarity index 89% rename from tools/gomod-required-updater/README.md rename to tools/gomod-local-update/README.md index 6549555e22f..23fa22e6314 100644 --- a/tools/gomod-required-updater/README.md +++ b/tools/gomod-local-update/README.md @@ -1,4 +1,4 @@ -# gomod-required-updater +# gomod-local-update Updates required module versions in go.mod files to match the latest git SHA from a remote branch. @@ -21,7 +21,7 @@ Optional: The installed binary will be placed in your `$GOPATH/bin` directory. Make sure this directory is in your system's PATH to run the command from anywhere. From the root of this repository, run: ```shell -go install ./tools/gomod-required-updater/cmd/gomod-required-updater +go install ./tools/gomod-local-update/cmd/gomod-local-update ``` ## Usage Examples diff --git a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go b/tools/gomod-local-update/cmd/gomod-local-update/main.go similarity index 61% rename from tools/gomod-required-updater/cmd/gomod-required-updater/main.go rename to tools/gomod-local-update/cmd/gomod-local-update/main.go index acb01779ee9..284c6a7e932 100644 --- a/tools/gomod-required-updater/cmd/gomod-required-updater/main.go +++ b/tools/gomod-local-update/cmd/gomod-local-update/main.go @@ -5,16 +5,19 @@ import ( "log" "os" - "github.com/smartcontractkit/chainlink/v2/tools/gomod-required-updater/internal/updater" + "github.com/smartcontractkit/chainlink/v2/tools/gomod-local-update/internal/updater" ) +const ( + goBinaryName = "gomod-local-update" +) var version = "dev" -var usage = `gomod-required-updater version %s +var usage = fmt.Sprintf(`%s version %%s Usage: cd /path/to/go/module - gomod-required-updater [flags] -` + %s [flags] +`, goBinaryName, goBinaryName) func main() { cfg, err := updater.ParseFlags(os.Args[1:], version) @@ -24,7 +27,7 @@ func main() { } if cfg.ShowVersion { - fmt.Printf("gomod-required-updater version %s\n", version) + fmt.Printf("%s version %s\n", goBinaryName, version) os.Exit(0) } diff --git a/tools/gomod-required-updater/internal/updater/config.go b/tools/gomod-local-update/internal/updater/config.go similarity index 95% rename from tools/gomod-required-updater/internal/updater/config.go rename to tools/gomod-local-update/internal/updater/config.go index ee14e9488a5..9173e2b2148 100644 --- a/tools/gomod-required-updater/internal/updater/config.go +++ b/tools/gomod-local-update/internal/updater/config.go @@ -23,7 +23,7 @@ type Config struct { } func ParseFlags(args []string, version string) (*Config, error) { - flags := flag.NewFlagSet("gomod-required-updater", flag.ContinueOnError) + flags := flag.NewFlagSet("default", flag.ContinueOnError) cfg := &Config{ ModulesToUpdate: make([]string, 0), diff --git a/tools/gomod-required-updater/internal/updater/config_test.go b/tools/gomod-local-update/internal/updater/config_test.go similarity index 100% rename from tools/gomod-required-updater/internal/updater/config_test.go rename to tools/gomod-local-update/internal/updater/config_test.go diff --git a/tools/gomod-required-updater/internal/updater/interfaces.go b/tools/gomod-local-update/internal/updater/interfaces.go similarity index 100% rename from tools/gomod-required-updater/internal/updater/interfaces.go rename to tools/gomod-local-update/internal/updater/interfaces.go diff --git a/tools/gomod-required-updater/internal/updater/module_operator.go b/tools/gomod-local-update/internal/updater/module_operator.go similarity index 100% rename from tools/gomod-required-updater/internal/updater/module_operator.go rename to tools/gomod-local-update/internal/updater/module_operator.go diff --git a/tools/gomod-required-updater/internal/updater/system_operator.go b/tools/gomod-local-update/internal/updater/system_operator.go similarity index 100% rename from tools/gomod-required-updater/internal/updater/system_operator.go rename to tools/gomod-local-update/internal/updater/system_operator.go diff --git a/tools/gomod-required-updater/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go similarity index 100% rename from tools/gomod-required-updater/internal/updater/updater.go rename to tools/gomod-local-update/internal/updater/updater.go diff --git a/tools/gomod-required-updater/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go similarity index 100% rename from tools/gomod-required-updater/internal/updater/updater_test.go rename to tools/gomod-local-update/internal/updater/updater_test.go From 9ec395a3bfc0692b29febeb79daf8e735c9aaa1d Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:54:40 -0500 Subject: [PATCH 28/47] Point to new binary --- .github/workflows/ci-gomod-required-updater.yml | 6 +++--- GNUmakefile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml index 570e94d23e8..420084b5457 100644 --- a/.github/workflows/ci-gomod-required-updater.yml +++ b/.github/workflows/ci-gomod-required-updater.yml @@ -1,4 +1,4 @@ -name: gomod-required-updater +name: gomod-local-update on: pull_request: @@ -12,5 +12,5 @@ jobs: uses: ./.github/actions/setup-go - name: Build binary run: | - go build -o dist/gomod-required-updater \ - ./tools/gomod-required-updater/cmd/gomod-required-updater + go build -o dist/gomod-local-update \ + ./tools/gomod-local-update/cmd/gomod-local-update diff --git a/GNUmakefile b/GNUmakefile index fd058a94aa3..d5f46461232 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -145,9 +145,9 @@ gomods: ## Install gomods go install github.com/jmank88/gomods@v0.1.4 .PHONY: gomodslocalupdate -gomodslocalupdate: ## Run gomod-required-updater - go install ./tools/gomod-required-updater/cmd/gomod-required-updater - gomods -w gomod-required-updater +gomodslocalupdate: ## Run gomod-local-update + go install ./tools/gomod-local-update/cmd/gomod-local-update + gomods -w gomod-local-update gomods tidy .PHONY: mockery From 72c98a4ec7d6c3c29a96402176bea6a818dcc422 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:35:13 -0500 Subject: [PATCH 29/47] Remove unused lint directive --- tools/gomod-local-update/internal/updater/module_operator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/gomod-local-update/internal/updater/module_operator.go b/tools/gomod-local-update/internal/updater/module_operator.go index ca57b5e8f8d..fe7c4ce9cda 100644 --- a/tools/gomod-local-update/internal/updater/module_operator.go +++ b/tools/gomod-local-update/internal/updater/module_operator.go @@ -88,7 +88,6 @@ func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, e return "", time.Time{}, fmt.Errorf("%w: empty SHA from git ls-remote", ErrModOperation) } - //nolint:gosec // SHA is obtained from git ls-remote output above cmd = exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) out, err = cmd.Output() if err != nil { From 938baead72e988a01b0a99be68c0032364c6e64b Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:44:35 -0500 Subject: [PATCH 30/47] Ran goimports -local --- tools/gomod-local-update/cmd/gomod-local-update/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/gomod-local-update/cmd/gomod-local-update/main.go b/tools/gomod-local-update/cmd/gomod-local-update/main.go index 284c6a7e932..7c486d667ef 100644 --- a/tools/gomod-local-update/cmd/gomod-local-update/main.go +++ b/tools/gomod-local-update/cmd/gomod-local-update/main.go @@ -11,6 +11,7 @@ import ( const ( goBinaryName = "gomod-local-update" ) + var version = "dev" var usage = fmt.Sprintf(`%s version %%s From a2b1834b52303b7d5483bc639d8e73c8aec98ba4 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:01:01 -0500 Subject: [PATCH 31/47] Cleanup and refactor --- .../cmd/gomod-local-update/main.go | 3 +- .../internal/updater/errors.go | 10 ++ .../internal/updater/interfaces.go | 44 ----- .../internal/updater/module_operator.go | 154 ------------------ .../internal/updater/system_operator.go | 8 + .../internal/updater/updater.go | 150 +++++++++++++---- .../internal/updater/updater_test.go | 125 +++----------- 7 files changed, 164 insertions(+), 330 deletions(-) create mode 100644 tools/gomod-local-update/internal/updater/errors.go delete mode 100644 tools/gomod-local-update/internal/updater/interfaces.go delete mode 100644 tools/gomod-local-update/internal/updater/module_operator.go diff --git a/tools/gomod-local-update/cmd/gomod-local-update/main.go b/tools/gomod-local-update/cmd/gomod-local-update/main.go index 7c486d667ef..d72cc2d0395 100644 --- a/tools/gomod-local-update/cmd/gomod-local-update/main.go +++ b/tools/gomod-local-update/cmd/gomod-local-update/main.go @@ -33,9 +33,8 @@ func main() { } u := updater.New( - updater.NewModuleOperator(cfg), - updater.NewSystemOperator(), cfg, + updater.NewSystemOperator(), ) if err := u.Run(); err != nil { diff --git a/tools/gomod-local-update/internal/updater/errors.go b/tools/gomod-local-update/internal/updater/errors.go new file mode 100644 index 00000000000..7b2f6bcc5b5 --- /dev/null +++ b/tools/gomod-local-update/internal/updater/errors.go @@ -0,0 +1,10 @@ +package updater + +import "errors" + +var ( + // ErrModOperation indicates a failure in a module-related operation + ErrModOperation = errors.New("module operation failed") + // ErrInvalidConfig indicates invalid configuration parameters + ErrInvalidConfig = errors.New("invalid configuration") +) \ No newline at end of file diff --git a/tools/gomod-local-update/internal/updater/interfaces.go b/tools/gomod-local-update/internal/updater/interfaces.go deleted file mode 100644 index f8dd222a038..00000000000 --- a/tools/gomod-local-update/internal/updater/interfaces.go +++ /dev/null @@ -1,44 +0,0 @@ -// Package updater provides functionality to update Go module versions in go.mod files. -// It specializes in handling local replace directives and maintaining consistent -// versioning across related modules. -package updater - -import ( - "errors" - "os" - "time" - - "golang.org/x/mod/modfile" - "golang.org/x/mod/module" -) - -const ( - gitSHALength = 12 -) - -// Errors that can be returned by the updater package -var ( - // ErrModOperation indicates a failure in a module-related operation - ErrModOperation = errors.New("module operation failed") - // ErrInvalidConfig indicates invalid configuration parameters - ErrInvalidConfig = errors.New("invalid configuration") -) - -// ModuleOperator handles Git repository operations and module version management. -// It provides functionality to retrieve Git information and manage module versions. -type ModuleOperator interface { - // GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository - GetGitInfo(remote, branch string) (string, time.Time, error) - // GetLatestVersion constructs a pseudo-version for a module based on Git information - GetLatestVersion(modulePath string) (module.Version, error) - // UpdateRequiredVersions identifies modules that need version updates - UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) -} - -// SystemOperator provides an interface for file system operations. -type SystemOperator interface { - // ReadFile reads the entire contents of a file - ReadFile(path string) ([]byte, error) - // WriteFile writes data to a file with specific permissions - WriteFile(path string, data []byte, perm os.FileMode) error -} diff --git a/tools/gomod-local-update/internal/updater/module_operator.go b/tools/gomod-local-update/internal/updater/module_operator.go deleted file mode 100644 index fe7c4ce9cda..00000000000 --- a/tools/gomod-local-update/internal/updater/module_operator.go +++ /dev/null @@ -1,154 +0,0 @@ -package updater - -import ( - "context" - "fmt" - "os/exec" - "regexp" - "strings" - "time" - - "golang.org/x/mod/modfile" - "golang.org/x/mod/module" -) - -const ( - majorVersionPattern = `/v\d+$` - gitTimeout = 30 * time.Second - gitTimeFormat = time.RFC3339 - gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` - gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` -) - -// getMajorVersion extracts the major version number from a module path -// Returns "v2" for /v2, "v1" for no version suffix -func getMajorVersion(modulePath string) string { - re := regexp.MustCompile(majorVersionPattern) - if match := re.FindString(modulePath); match != "" { - return strings.TrimPrefix(match, "/") - } - return "v0" -} - -type moduleOperator struct { - config *Config -} - -// NewModuleOperator creates a new ModuleOperator -func NewModuleOperator(config *Config) ModuleOperator { - if config.RepoRemote == "" { - config.RepoRemote = "origin" - } - if config.BranchTrunk == "" { - config.BranchTrunk = "develop" - } - return &moduleOperator{ - config: config, - } -} - -// validateGitInput checks if the remote and branch are in the correct format -func validateGitInput(remote, branch string) error { - remoteRE := regexp.MustCompile(gitRemotePattern) - if !remoteRE.MatchString(remote) { - return fmt.Errorf("%w: invalid git remote format", ErrModOperation) - } - - branchRE := regexp.MustCompile(gitBranchPattern) - if !branchRE.MatchString(branch) { - return fmt.Errorf("%w: invalid git branch format", ErrModOperation) - } - return nil -} - -// GetGitInfo retrieves the latest commit SHA and timestamp from a Git repository -func (m *moduleOperator) GetGitInfo(remote, branch string) (string, time.Time, error) { - // Validate remote and branch against strict regex patterns before using in exec - if err := validateGitInput(remote, branch); err != nil { - return "", time.Time{}, err - } - - ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) - defer cancel() - - // Safe to use remote/branch after validateGitInput ensures they match: - // - remote: ^[a-zA-Z0-9][-a-zA-Z0-9_.]*$ - // - branch: ^[a-zA-Z0-9][-a-zA-Z0-9/_]*$ - //nolint:gosec // Inputs are validated by regex patterns above - cmd := exec.CommandContext(ctx, "git", "ls-remote", remote, "refs/heads/"+branch) - out, err := cmd.Output() - if err != nil { - return "", time.Time{}, fmt.Errorf("%w: failed to get SHA: %w", ErrModOperation, err) - } - if len(out) == 0 { - return "", time.Time{}, fmt.Errorf("%w: no output from git ls-remote", ErrModOperation) - } - sha := strings.Split(string(out), "\t")[0] - if len(sha) == 0 { - return "", time.Time{}, fmt.Errorf("%w: empty SHA from git ls-remote", ErrModOperation) - } - - cmd = exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) - out, err = cmd.Output() - if err != nil { - return "", time.Time{}, fmt.Errorf("failed to get commit time: %w", err) - } - if len(out) == 0 { - return "", time.Time{}, fmt.Errorf("%w: no output from git show", ErrModOperation) - } - - commitTime, err := time.Parse(gitTimeFormat, strings.TrimSpace(string(out))) - if err != nil { - return "", time.Time{}, fmt.Errorf("failed to parse commit time: %w", err) - } - - return sha[:gitSHALength], commitTime, nil -} - -// GetLatestVersion retrieves the latest pseudo-version for a module -func (m *moduleOperator) GetLatestVersion(modulePath string) (module.Version, error) { - sha, commitTime, err := m.GetGitInfo(m.config.RepoRemote, m.config.BranchTrunk) - if err != nil { - return module.Version{}, err - } - - majorVer := strings.TrimPrefix(getMajorVersion(modulePath), "v") - pseudoVersion := module.PseudoVersion("v"+majorVer, "", commitTime, sha) - - return module.Version{ - Path: modulePath, - Version: pseudoVersion, - }, nil -} - -// extractVersionParts splits a pseudo-version into its components -func extractVersionParts(version string) (base, timestamp, sha string) { - parts := strings.Split(version, "-") - if len(parts) == 3 { - return parts[0], parts[1], parts[2] - } - return "", "", "" -} - -// UpdateRequiredVersions identifies modules that need version updates -func (m *moduleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) { - _, _, sha := extractVersionParts(newVersion) - if sha == "" { - return nil, fmt.Errorf("%w: invalid version format: %s", ErrModOperation, newVersion) - } - - var modulesToUpdate []string - localModules := make(map[string]bool) - for _, rep := range modFile.Replace { - if rep.New.Version == "" { - localModules[rep.Old.Path] = true - } - } - - for _, req := range modFile.Require { - if localModules[req.Mod.Path] { - modulesToUpdate = append(modulesToUpdate, req.Mod.Path) - } - } - return modulesToUpdate, nil -} diff --git a/tools/gomod-local-update/internal/updater/system_operator.go b/tools/gomod-local-update/internal/updater/system_operator.go index 49ed7ae4b40..59ccfb15ed7 100644 --- a/tools/gomod-local-update/internal/updater/system_operator.go +++ b/tools/gomod-local-update/internal/updater/system_operator.go @@ -5,6 +5,14 @@ import ( "path/filepath" ) +// SystemOperator provides an interface for file system operations. +type SystemOperator interface { + // ReadFile reads the entire contents of a file + ReadFile(path string) ([]byte, error) + // WriteFile writes data to a file with specific permissions + WriteFile(path string, data []byte, perm os.FileMode) error +} + type systemOperator struct{} func NewSystemOperator() SystemOperator { diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index b784765040a..6a5bf651908 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -1,9 +1,12 @@ package updater import ( + "context" "fmt" "log" "os" + "os/exec" + "regexp" "strings" "time" @@ -11,24 +14,95 @@ import ( "golang.org/x/mod/module" ) +// gitExecutor allows mocking git commands in tests +type gitExecutor interface { + Command(ctx context.Context, args ...string) ([]byte, error) +} + +// realGitExecutor implements actual git command execution +type realGitExecutor struct{} + +func (g *realGitExecutor) Command(ctx context.Context, args ...string) ([]byte, error) { + return exec.CommandContext(ctx, "git", args...).Output() +} + const ( - goModFile = "go.mod" - goModFileMode = 0644 + goModFile = "go.mod" + goModFileMode = 0644 + gitSHALength = 12 + gitTimeout = 30 * time.Second + gitTimeFormat = time.RFC3339 + gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` + gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` + majorVersionPattern = `/v\d+$` ) type Updater struct { - mod ModuleOperator - system SystemOperator config *Config + system SystemOperator + git gitExecutor } // New creates a new Updater -func New(mod ModuleOperator, system SystemOperator, config *Config) *Updater { - return &Updater{ - mod: mod, - system: system, - config: config, +func New(config *Config, system SystemOperator) *Updater { + return &Updater{ + config: config, + system: system, + git: &realGitExecutor{}, + } +} + +// validateGitInput checks if the remote and branch are in the correct format +func (u *Updater) validateGitInput(remote, branch string) error { + remoteRE := regexp.MustCompile(gitRemotePattern) + if !remoteRE.MatchString(remote) { + return fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) + } + + branchRE := regexp.MustCompile(gitBranchPattern) + if !branchRE.MatchString(branch) { + return fmt.Errorf("%w: git branch '%s' contains invalid characters", ErrInvalidConfig, branch) } + return nil +} + +// getGitInfo retrieves the latest commit SHA and timestamp from a Git repository +func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { + if err := u.validateGitInput(remote, branch); err != nil { + return "", time.Time{}, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) + defer cancel() + + out, err := u.git.Command(ctx, "ls-remote", remote, "refs/heads/"+branch) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: failed to fetch commit SHA from %s/%s: %v", + ErrModOperation, remote, branch, err) + } + if len(out) == 0 { + return "", time.Time{}, fmt.Errorf("%w: no output from git ls-remote", ErrModOperation) + } + sha := strings.Split(string(out), "\t")[0] + if len(sha) == 0 { + return "", time.Time{}, fmt.Errorf("%w: empty SHA from git ls-remote", ErrModOperation) + } + + cmd := exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) + out, err = cmd.Output() + if err != nil { + return "", time.Time{}, fmt.Errorf("failed to get commit time: %w", err) + } + if len(out) == 0 { + return "", time.Time{}, fmt.Errorf("%w: no output from git show", ErrModOperation) + } + + commitTime, err := time.Parse(gitTimeFormat, strings.TrimSpace(string(out))) + if err != nil { + return "", time.Time{}, fmt.Errorf("failed to parse commit time: %w", err) + } + + return sha[:gitSHALength], commitTime, nil } // Run starts the module update process @@ -43,14 +117,14 @@ func (u *Updater) Run() error { f, err := u.readModFile() if err != nil { - return err + return fmt.Errorf("%w: failed to read and parse go.mod file: %v", ErrModOperation, err) } // Find modules to update first if none specified if len(u.config.ModulesToUpdate) == 0 { u.config.ModulesToUpdate, err = u.findLocalReplaceModules() if err != nil { - return fmt.Errorf("failed to find local replace modules: %w", err) + return fmt.Errorf("%w: failed to detect local replace modules: %v", ErrModOperation, err) } if len(u.config.ModulesToUpdate) == 0 { logger.Printf("info: no modules found to update in %s", f.Module.Mod.Path) @@ -61,17 +135,16 @@ func (u *Updater) Run() error { } // Get commit info once for all modules - sha, commitTime, err := u.mod.GetGitInfo(u.config.RepoRemote, u.config.BranchTrunk) + sha, commitTime, err := u.getGitInfo(u.config.RepoRemote, u.config.BranchTrunk) if err != nil { - return fmt.Errorf("failed to get git info: %w", err) + return fmt.Errorf("%w: failed to get git commit info from remote: %v", ErrModOperation, err) } // Update the modules in the same file handle if err := u.updateGoMod(f, sha, commitTime); err != nil { - return fmt.Errorf("error updating %s: %w", goModFile, err) + return fmt.Errorf("%w: failed to update module versions in go.mod: %v", ErrModOperation, err) } - // Write the changes return u.writeModFile(f) } @@ -92,7 +165,8 @@ func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) } if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("failed to add requirement: %w", err) + return fmt.Errorf("%w: failed to update version for module %s: %v", + ErrModOperation, modulePath, err) } break } @@ -106,26 +180,40 @@ func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) return nil } +// getMajorVersion extracts the major version number from a module path +// Returns "v2" for /v2, "v1" for no version suffix +func getMajorVersion(modulePath string) string { + re := regexp.MustCompile(majorVersionPattern) + if match := re.FindString(modulePath); match != "" { + return strings.TrimPrefix(match, "/") + } + return "v0" +} + // findLocalReplaceModules finds modules with local replace directives func (u *Updater) findLocalReplaceModules() ([]string, error) { - var modules []string - seen := make(map[string]bool) - orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) - - // Use helper method instead of direct file read f, err := u.readModFile() if err != nil { return nil, err } + orgPrefix := fmt.Sprintf("github.com/%s/%s", u.config.OrgName, u.config.RepoName) + localModules := make(map[string]bool) + var modules []string + + // First find all local replaces for our org for _, rep := range f.Replace { - // Only process modules from our org/repo that have local replaces if strings.HasPrefix(rep.Old.Path, orgPrefix) && - isLocalPath(rep.New.Path) && - !seen[rep.Old.Path] { - log.Printf("Found local replace: %s => %s", rep.Old.Path, rep.New.Path) - modules = append(modules, rep.Old.Path) - seen[rep.Old.Path] = true + rep.New.Version == "" && + isLocalPath(rep.New.Path) { + localModules[rep.Old.Path] = true + } + } + + // Then check requires that match our replaces + for _, req := range f.Require { + if localModules[req.Mod.Path] { + modules = append(modules, req.Mod.Path) } } @@ -143,12 +231,12 @@ func isLocalPath(path string) bool { func (u *Updater) readModFile() (*modfile.File, error) { content, err := u.system.ReadFile(goModFile) if err != nil { - return nil, fmt.Errorf("failed to read %s: %w", goModFile, err) + return nil, fmt.Errorf("%w: unable to read go.mod: %v", ErrModOperation, err) } f, err := modfile.Parse(goModFile, content, nil) if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", goModFile, err) + return nil, fmt.Errorf("%w: invalid go.mod format: %v", ErrModOperation, err) } return f, nil @@ -158,11 +246,11 @@ func (u *Updater) readModFile() (*modfile.File, error) { func (u *Updater) writeModFile(f *modfile.File) error { content, err := f.Format() if err != nil { - return fmt.Errorf("failed to format %s: %w", goModFile, err) + return fmt.Errorf("%w: failed to format go.mod content: %v", ErrModOperation, err) } if err := u.system.WriteFile(goModFile, content, goModFileMode); err != nil { - return fmt.Errorf("failed to write %s: %w", goModFile, err) + return fmt.Errorf("%w: failed to write updated go.mod file: %v", ErrModOperation, err) } return nil diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index 06591ea91a0..bd373e4bd71 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -1,76 +1,28 @@ package updater import ( + "context" "fmt" "os" "testing" "time" - - "golang.org/x/mod/modfile" - "golang.org/x/mod/module" ) -// Mock implementations -type mockModuleOperator struct { - version module.Version - updateTime time.Time - org string - repo string - sha string - err error - modulesToUpdate []string -} - -func (m *mockModuleOperator) GetLatestVersion(modulePath string) (module.Version, error) { - if m.err != nil { - return module.Version{}, m.err - } - return m.version, nil -} - -func (m *mockModuleOperator) GetModuleInfo(modulePath string) (module.Version, time.Time, error) { - if m.err != nil { - return module.Version{}, time.Time{}, m.err - } - return m.version, m.updateTime, nil -} - -func (m *mockModuleOperator) ParseModulePathParts(modulePath string) (org, repo string, err error) { - if m.err != nil { - return "", "", m.err - } - if m.org == "" && m.repo == "" { - return "smartcontractkit", "chainlink", nil // Default values for tests - } - return m.org, m.repo, nil -} - -func (m *mockModuleOperator) GetGitInfo(remote, branch string) (string, time.Time, error) { - if m.err != nil { - return "", time.Time{}, m.err - } - if m.sha == "" { - m.sha = "abcdef123456" // default test SHA - } - return m.sha, m.updateTime, nil +// mockGitExecutor simulates git commands for testing +type mockGitExecutor struct { + sha string + time time.Time } -// Add the missing method -func (m *mockModuleOperator) UpdateRequiredVersions(modFile *modfile.File, newVersion string) ([]string, error) { - if m.err != nil { - return nil, m.err - } - if m.modulesToUpdate != nil { - return m.modulesToUpdate, nil - } - // Default behavior: return the modules that have local replace directives - var modules []string - for _, rep := range modFile.Replace { - if rep.New.Version == "" { // Local replace has empty version - modules = append(modules, rep.Old.Path) - } +func (m *mockGitExecutor) Command(ctx context.Context, args ...string) ([]byte, error) { + switch args[0] { + case "ls-remote": + return []byte(m.sha + "\trefs/heads/develop\n"), nil + case "show": + return []byte(m.time.Format(gitTimeFormat)), nil + default: + return nil, fmt.Errorf("unexpected git command: %v", args) } - return modules, nil } type mockSystemOperator struct { @@ -110,7 +62,6 @@ func TestUpdater_Run(t *testing.T) { tests := []struct { name string config *Config - modOp *mockModuleOperator sysOp *mockSystemOperator wantErr bool wantFile string @@ -119,13 +70,8 @@ func TestUpdater_Run(t *testing.T) { name: "successful update", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, - }, - modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/smartcontractkit/chainlink/v2", - Version: "v2.0.0", - }, - updateTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + RepoRemote: "origin", + BranchTrunk: "develop", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -140,12 +86,8 @@ require github.com/smartcontractkit/chainlink/v2 v2.0.0 name: "handles module with local replace", config: &Config{ ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, - }, - modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/smartcontractkit/chainlink/v2", - Version: "v2.0.0", - }, + RepoRemote: "origin", + BranchTrunk: "develop", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -161,12 +103,8 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ name: "v1 module update", config: &Config{ ModulesToUpdate: []string{"github.com/example/mod"}, - }, - modOp: &mockModuleOperator{ - version: module.Version{ - Path: "github.com/example/mod", - Version: "v1.2.3", - }, + RepoRemote: "origin", + BranchTrunk: "develop", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -184,10 +122,6 @@ require github.com/example/mod v1.0.0 RepoRemote: "origin", BranchTrunk: "develop", }, - modOp: &mockModuleOperator{ - sha: "ac7a7395feed", - updateTime: time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC), - }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`module test @@ -208,10 +142,6 @@ require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395f RepoRemote: "origin", BranchTrunk: "develop", }, - modOp: &mockModuleOperator{ - sha: testSHA, - updateTime: testTime, - }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`module test @@ -235,10 +165,6 @@ require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-a RepoRemote: "origin", BranchTrunk: "develop", }, - modOp: &mockModuleOperator{ - sha: testSHA, - updateTime: testTime, - }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`module test @@ -265,10 +191,6 @@ require ( RepoRemote: "origin", BranchTrunk: "develop", }, - modOp: &mockModuleOperator{ - sha: testSHA, - updateTime: testTime, - }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`module test @@ -286,7 +208,12 @@ require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241122182110-ac7a7395f for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - u := New(tt.modOp, tt.sysOp, tt.config) + u := New(tt.config, tt.sysOp) + // Override the git executor with our mock + u.git = &mockGitExecutor{ + sha: testSHA, + time: testTime, + } err := u.Run() if (err != nil) != tt.wantErr { t.Errorf("Updater.Run() error = %v, wantErr %v", err, tt.wantErr) @@ -320,7 +247,7 @@ replace ( RepoName: "chainlink", } - u := New(&mockModuleOperator{}, sysOp, cfg) + u := New(cfg, sysOp) modules, err := u.findLocalReplaceModules() if err != nil { t.Errorf("unexpected error: %v", err) From daa12bc2728d1b7122a680aeccd7ee863614cb7e Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:02:05 -0500 Subject: [PATCH 32/47] Format --- .../internal/updater/errors.go | 2 +- .../internal/updater/updater.go | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/errors.go b/tools/gomod-local-update/internal/updater/errors.go index 7b2f6bcc5b5..caf55535418 100644 --- a/tools/gomod-local-update/internal/updater/errors.go +++ b/tools/gomod-local-update/internal/updater/errors.go @@ -7,4 +7,4 @@ var ( ErrModOperation = errors.New("module operation failed") // ErrInvalidConfig indicates invalid configuration parameters ErrInvalidConfig = errors.New("invalid configuration") -) \ No newline at end of file +) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 6a5bf651908..3074711733f 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -16,24 +16,24 @@ import ( // gitExecutor allows mocking git commands in tests type gitExecutor interface { - Command(ctx context.Context, args ...string) ([]byte, error) + Command(ctx context.Context, args ...string) ([]byte, error) } // realGitExecutor implements actual git command execution type realGitExecutor struct{} func (g *realGitExecutor) Command(ctx context.Context, args ...string) ([]byte, error) { - return exec.CommandContext(ctx, "git", args...).Output() + return exec.CommandContext(ctx, "git", args...).Output() } const ( - goModFile = "go.mod" - goModFileMode = 0644 - gitSHALength = 12 - gitTimeout = 30 * time.Second - gitTimeFormat = time.RFC3339 - gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` - gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` + goModFile = "go.mod" + goModFileMode = 0644 + gitSHALength = 12 + gitTimeout = 30 * time.Second + gitTimeFormat = time.RFC3339 + gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` + gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` majorVersionPattern = `/v\d+$` ) @@ -45,11 +45,11 @@ type Updater struct { // New creates a new Updater func New(config *Config, system SystemOperator) *Updater { - return &Updater{ - config: config, - system: system, - git: &realGitExecutor{}, - } + return &Updater{ + config: config, + system: system, + git: &realGitExecutor{}, + } } // validateGitInput checks if the remote and branch are in the correct format @@ -77,7 +77,7 @@ func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { out, err := u.git.Command(ctx, "ls-remote", remote, "refs/heads/"+branch) if err != nil { - return "", time.Time{}, fmt.Errorf("%w: failed to fetch commit SHA from %s/%s: %v", + return "", time.Time{}, fmt.Errorf("%w: failed to fetch commit SHA from %s/%s: %v", ErrModOperation, remote, branch, err) } if len(out) == 0 { @@ -165,7 +165,7 @@ func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) } if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("%w: failed to update version for module %s: %v", + return fmt.Errorf("%w: failed to update version for module %s: %v", ErrModOperation, modulePath, err) } break From 98cbd7531885f386d8960b19e4e5ccce4d368e85 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:02:22 -0500 Subject: [PATCH 33/47] Update local requires --- core/scripts/go.mod | 4 ++-- deployment/go.mod | 2 +- integration-tests/go.mod | 4 ++-- integration-tests/load/go.mod | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 674b93d1ef0..e93335bce3f 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -10,8 +10,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122204838-a2678250f20b - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 ) require ( diff --git a/deployment/go.mod b/deployment/go.mod index b75f3809af8..c2abb95ffda 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -7,7 +7,7 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ // Using a separate inline `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 require ( github.com/Khan/genqlient v0.7.0 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 59ea7c43604..59c206a9eae 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -10,8 +10,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122204838-a2678250f20b - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 ) require ( diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index aea278bdc1c..8077e7ca5b7 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -12,9 +12,9 @@ replace github.com/smartcontractkit/chainlink/integration-tests => ../ // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122204838-a2678250f20b - github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241122204838-a2678250f20b - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122204838-a2678250f20b + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 ) require ( From df628489b525c727af344965b6e9ceebc377fc7c Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 29 Nov 2024 15:39:00 -0500 Subject: [PATCH 34/47] Add more refactoring --- tools/gomod-local-update/README.md | 17 +- .../cmd/gomod-local-update/main.go | 5 + .../internal/updater/config.go | 25 ++- .../internal/updater/errors.go | 5 +- .../internal/updater/updater.go | 95 +++++----- .../internal/updater/updater_test.go | 163 ++++++++++++++---- 6 files changed, 206 insertions(+), 104 deletions(-) diff --git a/tools/gomod-local-update/README.md b/tools/gomod-local-update/README.md index 23fa22e6314..56c6a966119 100644 --- a/tools/gomod-local-update/README.md +++ b/tools/gomod-local-update/README.md @@ -1,6 +1,6 @@ # gomod-local-update -Updates required module versions in go.mod files to match the latest git SHA from a remote branch. +Updates required module versions in `go.mod` files to match the latest git SHA from a remote branch. ## Configuration @@ -8,12 +8,11 @@ Command Line Flags: ```shell Optional: - -org-name Organization name to update modules for (default: smartcontractkit) - -repo-name Repository name to update modules for (default: chainlink) - -repo-remote Git remote to use (default: origin) - -branch-trunk Branch to get SHA from (default: develop) - -dry-run Preview changes without applying them (default: false) - - + -org-name Organization name (default: smartcontractkit) + -repo-name Repository name (default: chainlink) + -repo-remote Git remote to use (default: origin) + -branch-trunk Branch to get SHA from (default: develop) + -dry-run Preview changes without applying them (default: false) ``` ## Installation @@ -27,5 +26,7 @@ go install ./tools/gomod-local-update/cmd/gomod-local-update ## Usage Examples ```shell -make gomodslocalupdate +gomod-local-update ``` + +This command auto-detects modules with local replace directives and updates their versions. diff --git a/tools/gomod-local-update/cmd/gomod-local-update/main.go b/tools/gomod-local-update/cmd/gomod-local-update/main.go index d72cc2d0395..c471e8db776 100644 --- a/tools/gomod-local-update/cmd/gomod-local-update/main.go +++ b/tools/gomod-local-update/cmd/gomod-local-update/main.go @@ -32,6 +32,11 @@ func main() { os.Exit(0) } + if err := cfg.Validate(); err != nil { + fmt.Fprintf(os.Stderr, usage, version) + log.Fatal(err) + } + u := updater.New( cfg, updater.NewSystemOperator(), diff --git a/tools/gomod-local-update/internal/updater/config.go b/tools/gomod-local-update/internal/updater/config.go index 9173e2b2148..ed02961a7e3 100644 --- a/tools/gomod-local-update/internal/updater/config.go +++ b/tools/gomod-local-update/internal/updater/config.go @@ -13,21 +13,18 @@ const ( ) type Config struct { - ModulesToUpdate []string - RepoRemote string - BranchTrunk string - DryRun bool - ShowVersion bool - OrgName string - RepoName string + RepoRemote string + BranchTrunk string + DryRun bool + ShowVersion bool + OrgName string + RepoName string } func ParseFlags(args []string, version string) (*Config, error) { flags := flag.NewFlagSet("default", flag.ContinueOnError) - cfg := &Config{ - ModulesToUpdate: make([]string, 0), - } + cfg := &Config{} flags.StringVar(&cfg.RepoRemote, "repo-remote", DefaultRepoRemote, "Git remote to use") flags.StringVar(&cfg.BranchTrunk, "branch-trunk", DefaultBranchTrunk, "Branch to get SHA from") @@ -49,19 +46,19 @@ func (c *Config) Validate() error { } if c.RepoRemote == "" { - return fmt.Errorf("%w: repo-remote is required", ErrInvalidConfig) + return fmt.Errorf("%w: repo remote cannot be empty", ErrInvalidConfig) } if c.BranchTrunk == "" { - return fmt.Errorf("%w: branch-trunk is required", ErrInvalidConfig) + return fmt.Errorf("%w: branch trunk cannot be empty", ErrInvalidConfig) } if c.OrgName == "" { - return fmt.Errorf("%w: org-name is required", ErrInvalidConfig) + return fmt.Errorf("%w: organization name cannot be empty", ErrInvalidConfig) } if c.RepoName == "" { - return fmt.Errorf("%w: repo-name is required", ErrInvalidConfig) + return fmt.Errorf("%w: repository name cannot be empty", ErrInvalidConfig) } return nil diff --git a/tools/gomod-local-update/internal/updater/errors.go b/tools/gomod-local-update/internal/updater/errors.go index caf55535418..288eb49f66d 100644 --- a/tools/gomod-local-update/internal/updater/errors.go +++ b/tools/gomod-local-update/internal/updater/errors.go @@ -3,8 +3,9 @@ package updater import "errors" var ( - // ErrModOperation indicates a failure in a module-related operation + // ErrModOperation indicates a failure in a module-related operation. ErrModOperation = errors.New("module operation failed") - // ErrInvalidConfig indicates invalid configuration parameters + + // ErrInvalidConfig indicates invalid configuration parameters. ErrInvalidConfig = errors.New("invalid configuration") ) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 3074711733f..31a10e0971d 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -2,6 +2,7 @@ package updater import ( "context" + "errors" "fmt" "log" "os" @@ -35,6 +36,7 @@ const ( gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` majorVersionPattern = `/v\d+$` + shaPattern = `^[a-fA-F0-9]{40}$` // SHA-1 hashes are 40 hexadecimal characters ) type Updater struct { @@ -66,15 +68,25 @@ func (u *Updater) validateGitInput(remote, branch string) error { return nil } +// validateSHA checks if the SHA consists of exactly 40 hexadecimal digits +func (u *Updater) validateSHA(sha string) error { + shaRE := regexp.MustCompile(shaPattern) + if !shaRE.MatchString(sha) { + return fmt.Errorf("%w: invalid git SHA '%s'", ErrInvalidConfig, sha) + } + return nil +} + // getGitInfo retrieves the latest commit SHA and timestamp from a Git repository func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { if err := u.validateGitInput(remote, branch); err != nil { - return "", time.Time{}, fmt.Errorf("%w: %v", ErrInvalidConfig, err) + return "", time.Time{}, err } ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) defer cancel() + // Use u.git.Command for ls-remote out, err := u.git.Command(ctx, "ls-remote", remote, "refs/heads/"+branch) if err != nil { return "", time.Time{}, fmt.Errorf("%w: failed to fetch commit SHA from %s/%s: %v", @@ -88,14 +100,16 @@ func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { return "", time.Time{}, fmt.Errorf("%w: empty SHA from git ls-remote", ErrModOperation) } - cmd := exec.CommandContext(ctx, "git", "show", "-s", "--format=%cI", sha) - out, err = cmd.Output() + // Validate the SHA + if err := u.validateSHA(sha); err != nil { + return "", time.Time{}, err + } + + // Use u.git.Command for show + out, err = u.git.Command(ctx, "show", "-s", "--format=%cI", sha) if err != nil { return "", time.Time{}, fmt.Errorf("failed to get commit time: %w", err) } - if len(out) == 0 { - return "", time.Time{}, fmt.Errorf("%w: no output from git show", ErrModOperation) - } commitTime, err := time.Parse(gitTimeFormat, strings.TrimSpace(string(out))) if err != nil { @@ -109,54 +123,51 @@ func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { func (u *Updater) Run() error { logger := log.New(os.Stdout, "", log.LstdFlags) - if len(u.config.ModulesToUpdate) == 0 { - logger.Printf("info: auto-detecting modules with local replace directives") - } else { - logger.Printf("info: updating modules: %v", u.config.ModulesToUpdate) - } + logger.Printf("info: auto-detecting modules with local replace directives") f, err := u.readModFile() if err != nil { - return fmt.Errorf("%w: failed to read and parse go.mod file: %v", ErrModOperation, err) + return fmt.Errorf("%w: failed to read and parse go.mod file", ErrModOperation) } - // Find modules to update first if none specified - if len(u.config.ModulesToUpdate) == 0 { - u.config.ModulesToUpdate, err = u.findLocalReplaceModules() - if err != nil { - return fmt.Errorf("%w: failed to detect local replace modules: %v", ErrModOperation, err) - } - if len(u.config.ModulesToUpdate) == 0 { - logger.Printf("info: no modules found to update in %s", f.Module.Mod.Path) - return nil - } - logger.Printf("info: found %d modules with local replace directives: %v", - len(u.config.ModulesToUpdate), u.config.ModulesToUpdate) + // Auto-detect modules to update + modulesToUpdate, err := u.findLocalReplaceModules() + if err != nil { + return fmt.Errorf("%w: failed to detect local replace modules", ErrModOperation) } + if len(modulesToUpdate) == 0 { + logger.Printf("info: no modules found to update in %s", f.Module.Mod.Path) + return nil + } + logger.Printf("info: found %d modules with local replace directives: %v", + len(modulesToUpdate), modulesToUpdate) // Get commit info once for all modules sha, commitTime, err := u.getGitInfo(u.config.RepoRemote, u.config.BranchTrunk) if err != nil { - return fmt.Errorf("%w: failed to get git commit info from remote: %v", ErrModOperation, err) + if errors.Is(err, ErrInvalidConfig) { + return err + } + return fmt.Errorf("%w: failed to get git commit info from remote", ErrModOperation) } // Update the modules in the same file handle - if err := u.updateGoMod(f, sha, commitTime); err != nil { - return fmt.Errorf("%w: failed to update module versions in go.mod: %v", ErrModOperation, err) + if err := u.updateGoMod(f, modulesToUpdate, sha, commitTime); err != nil { + return fmt.Errorf("%w: failed to update module versions in go.mod", ErrModOperation) } return u.writeModFile(f) } // updateGoMod updates the go.mod file with new pseudo-versions -func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) error { - for _, modulePath := range u.config.ModulesToUpdate { +func (u *Updater) updateGoMod(modFile *modfile.File, modulesToUpdate []string, sha string, commitTime time.Time) error { + for _, modulePath := range modulesToUpdate { moduleExists := false majorVersion := getMajorVersion(modulePath) pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:gitSHALength]) // Find and update version - for _, req := range f.Require { + for _, req := range modFile.Require { if req.Mod.Path == modulePath { moduleExists = true if u.config.DryRun { @@ -164,9 +175,9 @@ func (u *Updater) updateGoMod(f *modfile.File, sha string, commitTime time.Time) continue } - if err := f.AddRequire(modulePath, pseudoVersion); err != nil { - return fmt.Errorf("%w: failed to update version for module %s: %v", - ErrModOperation, modulePath, err) + if err := modFile.AddRequire(modulePath, pseudoVersion); err != nil { + return fmt.Errorf("%w: failed to update version for module %s", + ErrModOperation, modulePath) } break } @@ -192,7 +203,7 @@ func getMajorVersion(modulePath string) string { // findLocalReplaceModules finds modules with local replace directives func (u *Updater) findLocalReplaceModules() ([]string, error) { - f, err := u.readModFile() + modFile, err := u.readModFile() if err != nil { return nil, err } @@ -202,7 +213,7 @@ func (u *Updater) findLocalReplaceModules() ([]string, error) { var modules []string // First find all local replaces for our org - for _, rep := range f.Replace { + for _, rep := range modFile.Replace { if strings.HasPrefix(rep.Old.Path, orgPrefix) && rep.New.Version == "" && isLocalPath(rep.New.Path) { @@ -211,7 +222,7 @@ func (u *Updater) findLocalReplaceModules() ([]string, error) { } // Then check requires that match our replaces - for _, req := range f.Require { + for _, req := range modFile.Require { if localModules[req.Mod.Path] { modules = append(modules, req.Mod.Path) } @@ -234,23 +245,23 @@ func (u *Updater) readModFile() (*modfile.File, error) { return nil, fmt.Errorf("%w: unable to read go.mod: %v", ErrModOperation, err) } - f, err := modfile.Parse(goModFile, content, nil) + modFile, err := modfile.Parse(goModFile, content, nil) if err != nil { return nil, fmt.Errorf("%w: invalid go.mod format: %v", ErrModOperation, err) } - return f, nil + return modFile, nil } // writeModFile writes the go.mod file -func (u *Updater) writeModFile(f *modfile.File) error { - content, err := f.Format() +func (u *Updater) writeModFile(modFile *modfile.File) error { + content, err := modFile.Format() if err != nil { - return fmt.Errorf("%w: failed to format go.mod content: %v", ErrModOperation, err) + return fmt.Errorf("%w: failed to format go.mod content", ErrModOperation) } if err := u.system.WriteFile(goModFile, content, goModFileMode); err != nil { - return fmt.Errorf("%w: failed to write updated go.mod file: %v", ErrModOperation, err) + return fmt.Errorf("%w: failed to write updated go.mod file", ErrModOperation) } return nil diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index bd373e4bd71..d845bba81a4 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -2,8 +2,10 @@ package updater import ( "context" + "errors" "fmt" "os" + "strings" "testing" "time" ) @@ -17,9 +19,27 @@ type mockGitExecutor struct { func (m *mockGitExecutor) Command(ctx context.Context, args ...string) ([]byte, error) { switch args[0] { case "ls-remote": - return []byte(m.sha + "\trefs/heads/develop\n"), nil + // Validate inputs + if len(args) != 3 { + return nil, fmt.Errorf("expected 3 args for ls-remote, got %d", len(args)) + } + remote := args[1] + if remote == "invalid*remote" { + return nil, fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) + } + // Return a full 40-character SHA + fullSHA := fmt.Sprintf("%s%s", m.sha, strings.Repeat("0", 40-len(m.sha))) + return []byte(fullSHA + "\trefs/heads/" + args[2] + "\n"), nil case "show": - return []byte(m.time.Format(gitTimeFormat)), nil + if len(args) != 4 { + return nil, fmt.Errorf("unexpected show args: %v", args) + } + // Use the full SHA for validation + fullSHA := fmt.Sprintf("%s%s", m.sha, strings.Repeat("0", 40-len(m.sha))) + if args[3] != fullSHA { + return nil, fmt.Errorf("unexpected SHA: got %s, want %s", args[3], fullSHA) + } + return []byte(m.time.Format(gitTimeFormat) + "\n"), nil default: return nil, fmt.Errorf("unexpected git command: %v", args) } @@ -57,7 +77,8 @@ func (m *mockSystemOperator) WriteFile(path string, data []byte, perm os.FileMod func TestUpdater_Run(t *testing.T) { testTime := time.Date(2024, 11, 22, 18, 21, 10, 0, time.UTC) - testSHA := "ac7a7395feed" + // Use a full 40-character SHA + testSHA := "ac7a7395feed" + strings.Repeat("0", 28) tests := []struct { name string @@ -69,25 +90,34 @@ func TestUpdater_Run(t *testing.T) { { name: "successful update", config: &Config{ - ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() m.files["go.mod"] = []byte(`module test require github.com/smartcontractkit/chainlink/v2 v2.0.0 +replace github.com/smartcontractkit/chainlink/v2 => ../ `) return m }(), wantErr: false, + wantFile: fmt.Sprintf(`module test + +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-%s + +replace github.com/smartcontractkit/chainlink/v2 => ../ +`, testSHA[:12]), }, { name: "handles module with local replace", config: &Config{ - ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -102,9 +132,10 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ { name: "v1 module update", config: &Config{ - ModulesToUpdate: []string{"github.com/example/mod"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", + OrgName: "example", + RepoName: "mod", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -118,9 +149,10 @@ require github.com/example/mod v1.0.0 { name: "updates v2 module with timestamp", config: &Config{ - ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v2"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -130,17 +162,20 @@ require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241119120536-03115e803 return m }(), wantErr: false, - wantFile: `module test + wantFile: fmt.Sprintf(`module test + +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-%s -require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed -`, +replace github.com/smartcontractkit/chainlink/v2 => ../ +`, testSHA[:12]), }, { name: "updates v0 module with timestamp", config: &Config{ - ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/deployment"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -150,20 +185,20 @@ require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241119120536-0 return m }(), wantErr: false, - wantFile: `module test + wantFile: fmt.Sprintf(`module test -require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed -`, +require github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-%s + +replace github.com/smartcontractkit/chainlink/deployment => ../ +`, testSHA[:12]), }, { name: "handles multiple modules with different versions", config: &Config{ - ModulesToUpdate: []string{ - "github.com/smartcontractkit/chainlink/v2", - "github.com/smartcontractkit/chainlink/deployment", - }, RepoRemote: "origin", BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -176,20 +211,25 @@ require ( return m }(), wantErr: false, - wantFile: `module test + wantFile: fmt.Sprintf(`module test require ( - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-ac7a7395feed - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-ac7a7395feed + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241122182110-%s + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241122182110-%s ) -`, + +replace github.com/smartcontractkit/chainlink/v2 => ../ + +replace github.com/smartcontractkit/chainlink/deployment => ../ +`, testSHA[:12], testSHA[:12]), }, { name: "updates v3 module with timestamp", config: &Config{ - ModulesToUpdate: []string{"github.com/smartcontractkit/chainlink/v3"}, - RepoRemote: "origin", - BranchTrunk: "develop", + RepoRemote: "origin", + BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", }, sysOp: func() *mockSystemOperator { m := newMockSystemOperator() @@ -199,16 +239,33 @@ require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241119120536-03115e803 return m }(), wantErr: false, - wantFile: `module test + wantFile: fmt.Sprintf(`module test + +require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241122182110-%s -require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241122182110-ac7a7395feed -`, +replace github.com/smartcontractkit/chainlink/v3 => ../ +`, testSHA[:12]), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { u := New(tt.config, tt.sysOp) + + // Add local replace directive for modules that should be updated + if !strings.Contains(tt.name, "v1 module update") { + modContent := string(tt.sysOp.files["go.mod"]) + for _, module := range []string{"v2", "v3", "deployment"} { + modulePath := fmt.Sprintf("github.com/%s/%s/%s", + tt.config.OrgName, tt.config.RepoName, module) + if strings.Contains(modContent, modulePath) && + !strings.Contains(modContent, "replace "+modulePath) { + modContent += fmt.Sprintf("\nreplace %s => ../\n", modulePath) + } + } + tt.sysOp.files["go.mod"] = []byte(modContent) + } + // Override the git executor with our mock u.git = &mockGitExecutor{ sha: testSHA, @@ -229,6 +286,36 @@ require github.com/smartcontractkit/chainlink/v3 v3.0.0-20241122182110-ac7a7395f } } +func TestUpdater_Run_InvalidGitInput(t *testing.T) { + cfg := &Config{ + RepoRemote: "invalid*remote", + BranchTrunk: "develop", + OrgName: "smartcontractkit", + RepoName: "chainlink", + } + sysOp := newMockSystemOperator() + sysOp.files["go.mod"] = []byte(`module test +require github.com/smartcontractkit/chainlink/v2 v2.0.0 +replace github.com/smartcontractkit/chainlink/v2 => ../ +`) + + u := New(cfg, sysOp) + u.git = &mockGitExecutor{ + sha: "ac7a7395feed" + strings.Repeat("0", 28), + time: time.Now(), + } + err := u.Run() + if err == nil { + t.Errorf("expected error due to invalid repo remote, got nil") + return + } + + // Use errors.Is instead of errors.As since ErrInvalidConfig is a sentinel error + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("expected error to be ErrInvalidConfig, got: %v", err) + } +} + func TestUpdater_FindLocalReplaceModules(t *testing.T) { sysOp := newMockSystemOperator() sysOp.files["go.mod"] = []byte(` From 9af0d0c3bb90e1ba746536debb9af9737ec42cd0 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 29 Nov 2024 15:51:20 -0500 Subject: [PATCH 35/47] Clarify readme --- tools/gomod-local-update/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gomod-local-update/README.md b/tools/gomod-local-update/README.md index 56c6a966119..d090aa1cdc6 100644 --- a/tools/gomod-local-update/README.md +++ b/tools/gomod-local-update/README.md @@ -1,6 +1,6 @@ # gomod-local-update -Updates required module versions in `go.mod` files to match the latest git SHA from a remote branch. +Updates any module that is `replace`'d with a local path to have its required module version in `go.mod` to match the latest git SHA from a remote branch. ## Configuration From 2f48978d90b0601a472dd02a2ecd739a808f061a Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 29 Nov 2024 15:54:52 -0500 Subject: [PATCH 36/47] Update comment --- tools/gomod-local-update/internal/updater/updater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 31a10e0971d..1bb1030fb36 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -192,7 +192,7 @@ func (u *Updater) updateGoMod(modFile *modfile.File, modulesToUpdate []string, s } // getMajorVersion extracts the major version number from a module path -// Returns "v2" for /v2, "v1" for no version suffix +// Returns "v2" for /v2, "v0" for no version suffix func getMajorVersion(modulePath string) string { re := regexp.MustCompile(majorVersionPattern) if match := re.FindString(modulePath); match != "" { From 63638a7989320530b0ffbc10c56d6372e3add0a8 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:18:48 -0500 Subject: [PATCH 37/47] Update README about usage and gomods --- tools/gomod-local-update/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/gomod-local-update/README.md b/tools/gomod-local-update/README.md index d090aa1cdc6..a49a663ecaa 100644 --- a/tools/gomod-local-update/README.md +++ b/tools/gomod-local-update/README.md @@ -2,6 +2,8 @@ Updates any module that is `replace`'d with a local path to have its required module version in `go.mod` to match the latest git SHA from a remote branch. +Is meant to run within each directory where a `go.mod` file is present. + ## Configuration Command Line Flags: @@ -25,8 +27,15 @@ go install ./tools/gomod-local-update/cmd/gomod-local-update ## Usage Examples +Run from the root of a go module directory. + ```shell gomod-local-update ``` -This command auto-detects modules with local replace directives and updates their versions. +Was designed to be used with [gomods](https://github.com/jmank88/gomods) like: + +```shell +gomods -w gomod-local-update +gomods tidy +``` From 88c1b7344540647f1465ca4de9315571267813c4 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:21:12 -0500 Subject: [PATCH 38/47] Add gomods as a dep for new target Co-authored-by: Jordan Krage --- GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index d5f46461232..336efd326a7 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -145,7 +145,7 @@ gomods: ## Install gomods go install github.com/jmank88/gomods@v0.1.4 .PHONY: gomodslocalupdate -gomodslocalupdate: ## Run gomod-local-update +gomodslocalupdate: gomods ## Run gomod-local-update go install ./tools/gomod-local-update/cmd/gomod-local-update gomods -w gomod-local-update gomods tidy From 941e0037181c5dd930de8e50d9ad6870d47a1812 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:31:55 -0500 Subject: [PATCH 39/47] Move pre-compiled regular expressions into vars --- .../internal/updater/updater.go | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 1bb1030fb36..607fd96b4e3 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -28,17 +28,28 @@ func (g *realGitExecutor) Command(ctx context.Context, args ...string) ([]byte, } const ( - goModFile = "go.mod" - goModFileMode = 0644 - gitSHALength = 12 - gitTimeout = 30 * time.Second - gitTimeFormat = time.RFC3339 + // File and mode constants + goModFile = "go.mod" + goModFileMode = 0644 + gitSHALength = 12 + gitTimeout = 30 * time.Second + gitTimeFormat = time.RFC3339 + + // Regex pattern constants gitRemotePattern = `^[a-zA-Z0-9][-a-zA-Z0-9_.]*$` gitBranchPattern = `^[a-zA-Z0-9][-a-zA-Z0-9/_]*$` majorVersionPattern = `/v\d+$` shaPattern = `^[a-fA-F0-9]{40}$` // SHA-1 hashes are 40 hexadecimal characters ) +var ( + // Pre-compiled regular expressions + gitRemoteRE = regexp.MustCompile(gitRemotePattern) + gitBranchRE = regexp.MustCompile(gitBranchPattern) + gitShaRE = regexp.MustCompile(shaPattern) + majorVersionRE = regexp.MustCompile(majorVersionPattern) +) + type Updater struct { config *Config system SystemOperator @@ -56,13 +67,11 @@ func New(config *Config, system SystemOperator) *Updater { // validateGitInput checks if the remote and branch are in the correct format func (u *Updater) validateGitInput(remote, branch string) error { - remoteRE := regexp.MustCompile(gitRemotePattern) - if !remoteRE.MatchString(remote) { + if !gitRemoteRE.MatchString(remote) { return fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) } - branchRE := regexp.MustCompile(gitBranchPattern) - if !branchRE.MatchString(branch) { + if !gitBranchRE.MatchString(branch) { return fmt.Errorf("%w: git branch '%s' contains invalid characters", ErrInvalidConfig, branch) } return nil @@ -70,8 +79,7 @@ func (u *Updater) validateGitInput(remote, branch string) error { // validateSHA checks if the SHA consists of exactly 40 hexadecimal digits func (u *Updater) validateSHA(sha string) error { - shaRE := regexp.MustCompile(shaPattern) - if !shaRE.MatchString(sha) { + if !gitShaRE.MatchString(sha) { return fmt.Errorf("%w: invalid git SHA '%s'", ErrInvalidConfig, sha) } return nil @@ -194,8 +202,7 @@ func (u *Updater) updateGoMod(modFile *modfile.File, modulesToUpdate []string, s // getMajorVersion extracts the major version number from a module path // Returns "v2" for /v2, "v0" for no version suffix func getMajorVersion(modulePath string) string { - re := regexp.MustCompile(majorVersionPattern) - if match := re.FindString(modulePath); match != "" { + if match := majorVersionRE.FindString(modulePath); match != "" { return strings.TrimPrefix(match, "/") } return "v0" From 833427af85bd0128bf62d31de3bfecc36240d691 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:33:54 -0500 Subject: [PATCH 40/47] Fix formatting --- tools/gomod-local-update/internal/updater/updater.go | 8 ++++---- tools/gomod-local-update/internal/updater/updater_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 607fd96b4e3..3ce21aba92b 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -44,10 +44,10 @@ const ( var ( // Pre-compiled regular expressions - gitRemoteRE = regexp.MustCompile(gitRemotePattern) - gitBranchRE = regexp.MustCompile(gitBranchPattern) - gitShaRE = regexp.MustCompile(shaPattern) - majorVersionRE = regexp.MustCompile(majorVersionPattern) + gitRemoteRE = regexp.MustCompile(gitRemotePattern) + gitBranchRE = regexp.MustCompile(gitBranchPattern) + gitShaRE = regexp.MustCompile(shaPattern) + majorVersionRE = regexp.MustCompile(majorVersionPattern) ) type Updater struct { diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index d845bba81a4..b7c848f2269 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -256,7 +256,7 @@ replace github.com/smartcontractkit/chainlink/v3 => ../ if !strings.Contains(tt.name, "v1 module update") { modContent := string(tt.sysOp.files["go.mod"]) for _, module := range []string{"v2", "v3", "deployment"} { - modulePath := fmt.Sprintf("github.com/%s/%s/%s", + modulePath := fmt.Sprintf("github.com/%s/%s/%s", tt.config.OrgName, tt.config.RepoName, module) if strings.Contains(modContent, modulePath) && !strings.Contains(modContent, "replace "+modulePath) { From 1d9497a42ab9874ac932b286659e90d6594df1d8 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:37:49 -0500 Subject: [PATCH 41/47] Remove remnant from refactor --- tools/gomod-local-update/internal/updater/updater.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 3ce21aba92b..268949c31a5 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -170,14 +170,12 @@ func (u *Updater) Run() error { // updateGoMod updates the go.mod file with new pseudo-versions func (u *Updater) updateGoMod(modFile *modfile.File, modulesToUpdate []string, sha string, commitTime time.Time) error { for _, modulePath := range modulesToUpdate { - moduleExists := false majorVersion := getMajorVersion(modulePath) pseudoVersion := module.PseudoVersion(majorVersion, "", commitTime, sha[:gitSHALength]) // Find and update version for _, req := range modFile.Require { if req.Mod.Path == modulePath { - moduleExists = true if u.config.DryRun { log.Printf("[DRY RUN] Would update %s: %s => %s", modulePath, req.Mod.Version, pseudoVersion) continue @@ -190,10 +188,6 @@ func (u *Updater) updateGoMod(modFile *modfile.File, modulesToUpdate []string, s break } } - - if !moduleExists { - continue - } } return nil From cbda2d1dfeb9d78e12f3dd7391e4e9da25379977 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:05:03 -0500 Subject: [PATCH 42/47] Move validation of git input into config --- .../internal/updater/config.go | 20 +++++---- .../internal/updater/config_test.go | 44 ++++++++++++++++++- .../internal/updater/updater.go | 16 ------- .../internal/updater/updater_test.go | 31 ------------- 4 files changed, 55 insertions(+), 56 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/config.go b/tools/gomod-local-update/internal/updater/config.go index ed02961a7e3..444193b8086 100644 --- a/tools/gomod-local-update/internal/updater/config.go +++ b/tools/gomod-local-update/internal/updater/config.go @@ -45,20 +45,24 @@ func (c *Config) Validate() error { return nil } + if c.OrgName == "" { + return fmt.Errorf("%w: org name must be provided", ErrInvalidConfig) + } + if c.RepoName == "" { + return fmt.Errorf("%w: repo name must be provided", ErrInvalidConfig) + } if c.RepoRemote == "" { - return fmt.Errorf("%w: repo remote cannot be empty", ErrInvalidConfig) + return fmt.Errorf("%w: repo remote must be provided", ErrInvalidConfig) } - if c.BranchTrunk == "" { - return fmt.Errorf("%w: branch trunk cannot be empty", ErrInvalidConfig) + return fmt.Errorf("%w: trunk branch must be provided", ErrInvalidConfig) } - if c.OrgName == "" { - return fmt.Errorf("%w: organization name cannot be empty", ErrInvalidConfig) + if !gitRemoteRE.MatchString(c.RepoRemote) { + return fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, c.RepoRemote) } - - if c.RepoName == "" { - return fmt.Errorf("%w: repository name cannot be empty", ErrInvalidConfig) + if !gitBranchRE.MatchString(c.BranchTrunk) { + return fmt.Errorf("%w: git branch '%s' contains invalid characters", ErrInvalidConfig, c.BranchTrunk) } return nil diff --git a/tools/gomod-local-update/internal/updater/config_test.go b/tools/gomod-local-update/internal/updater/config_test.go index 79fe72a4b9b..21b867aea8d 100644 --- a/tools/gomod-local-update/internal/updater/config_test.go +++ b/tools/gomod-local-update/internal/updater/config_test.go @@ -1,6 +1,9 @@ package updater -import "testing" +import ( + "errors" + "testing" +) func TestConfig_Validate(t *testing.T) { tests := []struct { @@ -61,6 +64,26 @@ func TestConfig_Validate(t *testing.T) { }, wantErr: true, }, + { + name: "invalid remote characters", + config: &Config{ + OrgName: "test", + RepoName: "test", + RepoRemote: "origin!@#", + BranchTrunk: "main", + }, + wantErr: true, + }, + { + name: "invalid branch characters", + config: &Config{ + OrgName: "test", + RepoName: "test", + RepoRemote: "origin", + BranchTrunk: "main!@#", + }, + wantErr: true, + }, } for _, tt := range tests { @@ -73,6 +96,25 @@ func TestConfig_Validate(t *testing.T) { } } +func TestConfig_ValidateErrorType(t *testing.T) { + cfg := &Config{ + RepoRemote: "invalid*remote", + BranchTrunk: "develop", + OrgName: "test", + RepoName: "test", + } + + err := cfg.Validate() + if err == nil { + t.Error("expected error due to invalid repo remote, got nil") + return + } + + if !errors.Is(err, ErrInvalidConfig) { + t.Errorf("expected error to be ErrInvalidConfig, got: %v", err) + } +} + func TestParseFlags(t *testing.T) { tests := []struct { name string diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 268949c31a5..d44e144192f 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -65,18 +65,6 @@ func New(config *Config, system SystemOperator) *Updater { } } -// validateGitInput checks if the remote and branch are in the correct format -func (u *Updater) validateGitInput(remote, branch string) error { - if !gitRemoteRE.MatchString(remote) { - return fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) - } - - if !gitBranchRE.MatchString(branch) { - return fmt.Errorf("%w: git branch '%s' contains invalid characters", ErrInvalidConfig, branch) - } - return nil -} - // validateSHA checks if the SHA consists of exactly 40 hexadecimal digits func (u *Updater) validateSHA(sha string) error { if !gitShaRE.MatchString(sha) { @@ -87,10 +75,6 @@ func (u *Updater) validateSHA(sha string) error { // getGitInfo retrieves the latest commit SHA and timestamp from a Git repository func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { - if err := u.validateGitInput(remote, branch); err != nil { - return "", time.Time{}, err - } - ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) defer cancel() diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index b7c848f2269..da1af2f97e0 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -2,7 +2,6 @@ package updater import ( "context" - "errors" "fmt" "os" "strings" @@ -286,36 +285,6 @@ replace github.com/smartcontractkit/chainlink/v3 => ../ } } -func TestUpdater_Run_InvalidGitInput(t *testing.T) { - cfg := &Config{ - RepoRemote: "invalid*remote", - BranchTrunk: "develop", - OrgName: "smartcontractkit", - RepoName: "chainlink", - } - sysOp := newMockSystemOperator() - sysOp.files["go.mod"] = []byte(`module test -require github.com/smartcontractkit/chainlink/v2 v2.0.0 -replace github.com/smartcontractkit/chainlink/v2 => ../ -`) - - u := New(cfg, sysOp) - u.git = &mockGitExecutor{ - sha: "ac7a7395feed" + strings.Repeat("0", 28), - time: time.Now(), - } - err := u.Run() - if err == nil { - t.Errorf("expected error due to invalid repo remote, got nil") - return - } - - // Use errors.Is instead of errors.As since ErrInvalidConfig is a sentinel error - if !errors.Is(err, ErrInvalidConfig) { - t.Errorf("expected error to be ErrInvalidConfig, got: %v", err) - } -} - func TestUpdater_FindLocalReplaceModules(t *testing.T) { sysOp := newMockSystemOperator() sysOp.files["go.mod"] = []byte(` From b80bcfaff198a596af2736a83fd82ccf8e5da742 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:09:12 -0500 Subject: [PATCH 43/47] Run `make gomodslocalupdate` --- core/scripts/go.mod | 4 ++-- deployment/go.mod | 2 +- integration-tests/go.mod | 4 ++-- integration-tests/load/go.mod | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/scripts/go.mod b/core/scripts/go.mod index a080b2efc58..7bf43089bee 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -12,8 +12,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241129175708-44cab8dc1544 - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241206210521-125d98cdaf66 + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241206210521-125d98cdaf66 ) require ( diff --git a/deployment/go.mod b/deployment/go.mod index 9f43184146a..13b57de7d47 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -9,7 +9,7 @@ replace github.com/smartcontractkit/chainlink/v2 => ../ // Using a separate inline `require` here to avoid surrounding line changes // creating potential merge conflicts. -require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 +require github.com/smartcontractkit/chainlink/v2 v2.0.0-20241206210521-125d98cdaf66 require ( github.com/Khan/genqlient v0.7.0 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 95cff1166e0..8dae53c4aac 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -12,8 +12,8 @@ replace github.com/smartcontractkit/chainlink/deployment => ../deployment // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241129175708-44cab8dc1544 - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241206210521-125d98cdaf66 + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241206210521-125d98cdaf66 ) require ( diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index af5c2d494f2..0235f0389f5 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -14,9 +14,9 @@ replace github.com/smartcontractkit/chainlink/integration-tests => ../ // Using a separate `require` here to avoid surrounding line changes // creating potential merge conflicts. require ( - github.com/smartcontractkit/chainlink/deployment v0.0.0-20241129175708-44cab8dc1544 - github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241129175708-44cab8dc1544 - github.com/smartcontractkit/chainlink/v2 v2.0.0-20241129175708-44cab8dc1544 + github.com/smartcontractkit/chainlink/deployment v0.0.0-20241206210521-125d98cdaf66 + github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241206210521-125d98cdaf66 + github.com/smartcontractkit/chainlink/v2 v2.0.0-20241206210521-125d98cdaf66 ) require ( From 89dd71b6d77e63a5430e68d8e10506a8dec86e63 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 6 Dec 2024 19:09:28 -0500 Subject: [PATCH 44/47] Appease linter --- .../internal/updater/updater.go | 18 +++++++++--------- .../internal/updater/updater_test.go | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index d44e144192f..44c99fb273c 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -81,7 +81,7 @@ func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { // Use u.git.Command for ls-remote out, err := u.git.Command(ctx, "ls-remote", remote, "refs/heads/"+branch) if err != nil { - return "", time.Time{}, fmt.Errorf("%w: failed to fetch commit SHA from %s/%s: %v", + return "", time.Time{}, fmt.Errorf("%w: failed to fetch commit SHA from %s/%s: %w", ErrModOperation, remote, branch, err) } if len(out) == 0 { @@ -98,14 +98,14 @@ func (u *Updater) getGitInfo(remote, branch string) (string, time.Time, error) { } // Use u.git.Command for show - out, err = u.git.Command(ctx, "show", "-s", "--format=%cI", sha) - if err != nil { - return "", time.Time{}, fmt.Errorf("failed to get commit time: %w", err) + showOut, showErr := u.git.Command(ctx, "show", "-s", "--format=%cI", sha) + if showErr != nil { + return "", time.Time{}, fmt.Errorf("failed to get commit time: %w", showErr) } - commitTime, err := time.Parse(gitTimeFormat, strings.TrimSpace(string(out))) - if err != nil { - return "", time.Time{}, fmt.Errorf("failed to parse commit time: %w", err) + commitTime, parseErr := time.Parse(gitTimeFormat, strings.TrimSpace(string(showOut))) + if parseErr != nil { + return "", time.Time{}, fmt.Errorf("failed to parse commit time: %w", parseErr) } return sha[:gitSHALength], commitTime, nil @@ -227,12 +227,12 @@ func isLocalPath(path string) bool { func (u *Updater) readModFile() (*modfile.File, error) { content, err := u.system.ReadFile(goModFile) if err != nil { - return nil, fmt.Errorf("%w: unable to read go.mod: %v", ErrModOperation, err) + return nil, fmt.Errorf("unable to read go.mod: %w", err) } modFile, err := modfile.Parse(goModFile, content, nil) if err != nil { - return nil, fmt.Errorf("%w: invalid go.mod format: %v", ErrModOperation, err) + return nil, fmt.Errorf("%w: invalid go.mod format: %w", ErrModOperation, err) // Changed %v to %w } return modFile, nil diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index da1af2f97e0..a08e17ddd7e 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -24,7 +24,7 @@ func (m *mockGitExecutor) Command(ctx context.Context, args ...string) ([]byte, } remote := args[1] if remote == "invalid*remote" { - return nil, fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) + return nil, fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) // Fixed: added %w } // Return a full 40-character SHA fullSHA := fmt.Sprintf("%s%s", m.sha, strings.Repeat("0", 40-len(m.sha))) From 25864d169c4676f416e46683b21b2b23cb5a71cd Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Sat, 7 Dec 2024 12:15:29 -0500 Subject: [PATCH 45/47] Remove unnecessary inline comments --- tools/gomod-local-update/internal/updater/updater.go | 2 +- tools/gomod-local-update/internal/updater/updater_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index 44c99fb273c..ffd4814d5f4 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -232,7 +232,7 @@ func (u *Updater) readModFile() (*modfile.File, error) { modFile, err := modfile.Parse(goModFile, content, nil) if err != nil { - return nil, fmt.Errorf("%w: invalid go.mod format: %w", ErrModOperation, err) // Changed %v to %w + return nil, fmt.Errorf("%w: invalid go.mod format: %w", ErrModOperation, err) } return modFile, nil diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index a08e17ddd7e..da1af2f97e0 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -24,7 +24,7 @@ func (m *mockGitExecutor) Command(ctx context.Context, args ...string) ([]byte, } remote := args[1] if remote == "invalid*remote" { - return nil, fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) // Fixed: added %w + return nil, fmt.Errorf("%w: git remote '%s' contains invalid characters", ErrInvalidConfig, remote) } // Return a full 40-character SHA fullSHA := fmt.Sprintf("%s%s", m.sha, strings.Repeat("0", 40-len(m.sha))) From 4be3d9466992d639511e88698b41aa47508d9d45 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:37:09 -0500 Subject: [PATCH 46/47] Use a better name for git exec --- tools/gomod-local-update/internal/updater/updater.go | 8 ++++---- tools/gomod-local-update/internal/updater/updater_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/gomod-local-update/internal/updater/updater.go b/tools/gomod-local-update/internal/updater/updater.go index ffd4814d5f4..cfb873ed3ca 100644 --- a/tools/gomod-local-update/internal/updater/updater.go +++ b/tools/gomod-local-update/internal/updater/updater.go @@ -20,10 +20,10 @@ type gitExecutor interface { Command(ctx context.Context, args ...string) ([]byte, error) } -// realGitExecutor implements actual git command execution -type realGitExecutor struct{} +// systemGitExecutor executes git commands on the host system +type systemGitExecutor struct{} -func (g *realGitExecutor) Command(ctx context.Context, args ...string) ([]byte, error) { +func (g *systemGitExecutor) Command(ctx context.Context, args ...string) ([]byte, error) { return exec.CommandContext(ctx, "git", args...).Output() } @@ -61,7 +61,7 @@ func New(config *Config, system SystemOperator) *Updater { return &Updater{ config: config, system: system, - git: &realGitExecutor{}, + git: &systemGitExecutor{}, } } diff --git a/tools/gomod-local-update/internal/updater/updater_test.go b/tools/gomod-local-update/internal/updater/updater_test.go index da1af2f97e0..ba17024dd37 100644 --- a/tools/gomod-local-update/internal/updater/updater_test.go +++ b/tools/gomod-local-update/internal/updater/updater_test.go @@ -9,7 +9,7 @@ import ( "time" ) -// mockGitExecutor simulates git commands for testing +// mockGitExecutor simulates git commands for testing as an alternative to systemGitExecutor type mockGitExecutor struct { sha string time time.Time From ccf2842be4413d41cb4141fa7bc4d4e3dc4c3a0d Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:46:11 -0500 Subject: [PATCH 47/47] Remove unnecessary build workflow --- .github/workflows/ci-gomod-required-updater.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .github/workflows/ci-gomod-required-updater.yml diff --git a/.github/workflows/ci-gomod-required-updater.yml b/.github/workflows/ci-gomod-required-updater.yml deleted file mode 100644 index 420084b5457..00000000000 --- a/.github/workflows/ci-gomod-required-updater.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: gomod-local-update - -on: - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.2.1 - - name: Setup Go - uses: ./.github/actions/setup-go - - name: Build binary - run: | - go build -o dist/gomod-local-update \ - ./tools/gomod-local-update/cmd/gomod-local-update