From 945b81c8c97f1158a1c0f71d3e923b8879c436c3 Mon Sep 17 00:00:00 2001 From: Aleksandr Tarasov Date: Thu, 16 Nov 2023 17:54:59 +0300 Subject: [PATCH] Add option patcher to smooth compilation (#7) --- Makefile | 5 + README.md | 12 ++- go.mod | 1 + go.sum | 2 + internal/git/vendor.go | 25 ++++- internal/model/model.go | 13 +-- internal/modules/modules.go | 27 +++++- internal/patcher/gopackage.go | 52 ++++++++++ internal/patcher/gopackage_test.go | 101 ++++++++++++++++++++ internal/patcher/patcher.go | 16 ++++ internal/registry/vendor.go | 24 ++++- pbuf.yaml | 1 + third_party/pbuf-registry/v1/entities.proto | 2 +- third_party/pbuf-registry/v1/registry.proto | 2 +- 14 files changed, 267 insertions(+), 16 deletions(-) create mode 100644 internal/patcher/gopackage.go create mode 100644 internal/patcher/gopackage_test.go create mode 100644 internal/patcher/patcher.go diff --git a/Makefile b/Makefile index d47491f..28db27f 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,11 @@ docker: lint: golangci-lint run -v --timeout 10m +.PHONE: test +# test +test: + go test ./... -cover + # show help help: @echo '' diff --git a/README.md b/README.md index abbb8c0..6046e7d 100644 --- a/README.md +++ b/README.md @@ -167,12 +167,14 @@ modules: path: [path_in_registry] tag: [dependency_module_tag] out: [output_folder_on_local] + gen_out: [gen_output_folder_on_local] # optional, if provided then patchers will be applied # use a git repository to vendor .proto files - repository: [repository_url] path: [path_in_repository] branch: [branch_name] tag: [tag_name] out: [output_folder_on_local] + gen_out: [gen_output_folder_on_local] # optional, if provided then patchers will be applied ``` Replace main placeholders with appropriate values: @@ -185,6 +187,7 @@ Replace placeholders in the registry modules with appropriate values: - `[path_in_registry]`: Path to the folder or file in the registry you want to vendor. - `[tag_name]`: Specific tag to vendor. - `[output_folder_on_local]`: Folder where the vendor content should be placed on your local machine. +- `[gen_output_folder_on_local]`: Folder where the generated content should be placed on your local machine. Used to patch `go_package` option Replace placeholders in modules placed in git with appropriate values: - `[repository_url]`: The URL of the Git repository. @@ -192,6 +195,7 @@ Replace placeholders in modules placed in git with appropriate values: - `[branch_name]`: Specific branch name to clone (optional if tag is provided). - `[tag_name]`: Specific tag to clone (optional if branch is provided). - `[output_folder_on_local]`: Folder where the vendor content should be placed on your local machine. +- `[gen_output_folder_on_local]`: Folder where the generated content should be placed on your local machine. Used to patch `go_package` option #### Examples @@ -219,10 +223,16 @@ registry: addr: pbuf.cloud:8081 insecure: true modules: - # will copy api/v1/registry.proto file to third_party/api/v1/registry.proto + # will copy api/v1/*.proto file to third_party/api/v1/*.proto - name: pbufio/pbuf-registry tag: v0.0.1 out: third_party + # will copy api/v1/*.proto file to third_party/api/v1/*.proto + # and add or change `go_package` option to `/gen/pbuf-registry/api/v1` + - name: pbufio/pbuf-registry + tag: v0.0.1 + out: third_party + gen_out: gen # will copy examples/addressbook.proto file to proto/addressbook.proto - repository: https://github.com/protocolbuffers/protobuf path: examples/addressbook.proto diff --git a/go.mod b/go.mod index cce68c7..7bd9a72 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-git/go-git/v5 v5.9.0 github.com/jdx/go-netrc v1.0.0 github.com/spf13/cobra v1.7.0 + github.com/yoheimuta/go-protoparser/v4 v4.9.0 google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 diff --git a/go.sum b/go.sum index b52e4e7..b88a1de 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yoheimuta/go-protoparser/v4 v4.9.0 h1:zHRXzRjkOamwMkPu7bpiCtOpxHkM9c8zxQOvW99eWlo= +github.com/yoheimuta/go-protoparser/v4 v4.9.0/go.mod h1:AHNNnSWnb0UoL4QgHPiOAg2BniQceFscPI5X/BZNHl8= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/internal/git/vendor.go b/internal/git/vendor.go index 1ca4589..e4341b8 100644 --- a/internal/git/vendor.go +++ b/internal/git/vendor.go @@ -17,9 +17,10 @@ import ( "github.com/go-git/go-git/v5/storage/memory" "github.com/jdx/go-netrc" "github.com/pbufio/pbuf-cli/internal/model" + "github.com/pbufio/pbuf-cli/internal/patcher" ) -func VendorGitModule(module *model.Module, netrcAuth *netrc.Netrc) error { +func VendorGitModule(module *model.Module, netrcAuth *netrc.Netrc, patchers []patcher.Patcher) error { log.Printf("start vendoring .proto files. repo: %s, path: %s", module.Repository, module.Path) var reference plumbing.ReferenceName @@ -118,13 +119,33 @@ func VendorGitModule(module *model.Module, netrcAuth *netrc.Netrc) error { return err } + var content string + outputDir := filepath.Dir(outputPath) + + if module.GenerateOutputFolder != "" { + content, err = patcher.ApplyPatchers( + patchers, + strings.Replace(outputDir, module.OutputFolder, module.GenerateOutputFolder, 1), + string(fileContents), + ) + if err != nil { + log.Printf("failed to patch file %s: %v", outputPath, err) + } + } else { + content = string(fileContents) + } + + if err != nil { + log.Printf("failed to patch file %s: %v", outputPath, err) + } + copiedFile, err := os.Create(outputPath) if err != nil { log.Printf("failed to create file: %s", outputPath) return err } - _, err = copiedFile.Write(fileContents) + _, err = copiedFile.Write([]byte(content)) if err != nil { log.Printf("failed to write file contents: %s", outputPath) return err diff --git a/internal/model/model.go b/internal/model/model.go index d95a00a..ea19e6f 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -14,12 +14,13 @@ type Config struct { } type Module struct { - Name string `yaml:"name"` - Repository string `yaml:"repository"` - Path string `yaml:"path"` - Branch string `yaml:"branch"` - Tag string `yaml:"tag"` - OutputFolder string `yaml:"out"` + Name string `yaml:"name"` + Repository string `yaml:"repository"` + Path string `yaml:"path"` + Branch string `yaml:"branch"` + Tag string `yaml:"tag"` + OutputFolder string `yaml:"out"` + GenerateOutputFolder string `yaml:"gen_out"` } func (c *Config) HasRegistry() bool { diff --git a/internal/modules/modules.go b/internal/modules/modules.go index 3a88fa8..a50ef61 100644 --- a/internal/modules/modules.go +++ b/internal/modules/modules.go @@ -2,6 +2,7 @@ package modules import ( "log" + "os" "os/user" "path/filepath" @@ -9,7 +10,9 @@ import ( v1 "github.com/pbufio/pbuf-cli/gen/pbuf-registry/v1" "github.com/pbufio/pbuf-cli/internal/git" "github.com/pbufio/pbuf-cli/internal/model" + "github.com/pbufio/pbuf-cli/internal/patcher" "github.com/pbufio/pbuf-cli/internal/registry" + "golang.org/x/mod/modfile" "gopkg.in/yaml.v2" ) @@ -23,6 +26,24 @@ func NewConfig(contents []byte) (*model.Config, error) { return modulesConfig, nil } +// newProtoPatchers create a slice of proto patchers +func newProtoPatchers() []patcher.Patcher { + var result []patcher.Patcher + + // if we have go.mod file + // then parse it and fetch the module name + // and pass it to the go package patcher + file, err := os.ReadFile("go.mod") + if err == nil { + // that's ok, we cannot find go mod file + path := modfile.ModulePath(file) + if path != "" { + result = append(result, patcher.NewGoPackagePatcher(path)) + } + } + return result +} + // Vendor function that iterate over the modules and vendor proto files from git repositories func Vendor(config *model.Config, client v1.RegistryClient) error { usr, err := user.Current() @@ -36,6 +57,8 @@ func Vendor(config *model.Config, client v1.RegistryClient) error { log.Printf("no .netrc file found. skipping auth") } + patchers := newProtoPatchers() + for _, module := range config.Modules { if module.Repository == "" { if config.HasRegistry() { @@ -47,7 +70,7 @@ func Vendor(config *model.Config, client v1.RegistryClient) error { log.Fatalf("no module tag found for module: %v", module) } - err := registry.VendorRegistryModule(module, client) + err := registry.VendorRegistryModule(module, client, patchers) if err != nil { log.Fatalf("failed to vendor module %s: %v", module.Name, err) } @@ -55,7 +78,7 @@ func Vendor(config *model.Config, client v1.RegistryClient) error { log.Fatalf("no repository found for module: %s", module.Name) } } else { - err := git.VendorGitModule(module, netrcAuth) + err := git.VendorGitModule(module, netrcAuth, patchers) if err != nil { log.Fatalf("failed to vendor module %s: %v", module.Repository, err) } diff --git a/internal/patcher/gopackage.go b/internal/patcher/gopackage.go new file mode 100644 index 0000000..a716c0a --- /dev/null +++ b/internal/patcher/gopackage.go @@ -0,0 +1,52 @@ +package patcher + +import ( + "fmt" + "strings" + + "github.com/yoheimuta/go-protoparser/v4/interpret/unordered" +) +import "github.com/yoheimuta/go-protoparser/v4" + +type GoPackagePatcher struct { + goModule string +} + +func NewGoPackagePatcher(goModule string) *GoPackagePatcher { + return &GoPackagePatcher{ + goModule: goModule, + } +} + +func (p *GoPackagePatcher) Patch(outputPath, content string) (string, error) { + parsed, err := protoparser.Parse(strings.NewReader(content)) + if err != nil { + return "", err + } + + proto, err := unordered.InterpretProto(parsed) + if err != nil { + return "", err + } + + dirs := strings.Split(outputPath, "/") + goPackage := fmt.Sprintf(`option go_package = "%s/%s;%s";`, p.goModule, outputPath, dirs[len(dirs)-1]) + + for _, option := range proto.ProtoBody.Options { + if option.OptionName == "go_package" { + // break by lines + // option.Meta.Pos.Line as the line to change + splitted := strings.Split(content, "\n") + splitted[option.Meta.Pos.Line-1] = goPackage + return strings.Join(splitted, "\n"), nil + } + } + + // if no go_package option, add it + splitted := strings.Split(content, "\n") + // add the element after syntax line + line := proto.Syntax.Meta.LastPos.Line + splitted = append(splitted[:line], append([]string{goPackage}, splitted[line:]...)...) + + return strings.Join(splitted, "\n"), nil +} diff --git a/internal/patcher/gopackage_test.go b/internal/patcher/gopackage_test.go new file mode 100644 index 0000000..c9f3a7e --- /dev/null +++ b/internal/patcher/gopackage_test.go @@ -0,0 +1,101 @@ +package patcher + +import "testing" + +const ( + noPackageProtoFile = ` +syntax = "proto3"; +package pbufregistry.v1; + +// Module is a module registered in the registry. +message Module { +} +` + noPackageProtoFilePatched = ` +syntax = "proto3"; +option go_package = "github.com/pbufio/pbuf-registry/api/v1;v1"; +package pbufregistry.v1; + +// Module is a module registered in the registry. +message Module { +} +` + + withPackageProtoFile = ` +syntax = "proto3"; +package pbufregistry.v1; + +option go_package = "pbufregistry/api/v2;v2"; + +// Module is a module registered in the registry. +message Module { +} +` + + withPackageProtoFilePatched = ` +syntax = "proto3"; +package pbufregistry.v1; + +option go_package = "github.com/pbufio/pbuf-registry/api/v1;v1"; + +// Module is a module registered in the registry. +message Module { +} +` +) + +func TestGoPackagePatcher_Patch(t *testing.T) { + type fields struct { + goModule string + } + type args struct { + outputPath string + content string + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "no package", + fields: fields{ + goModule: "github.com/pbufio/pbuf-registry", + }, + args: args{ + outputPath: "api/v1", + content: noPackageProtoFile, + }, + want: noPackageProtoFilePatched, + wantErr: false, + }, + { + name: "with package", + fields: fields{ + goModule: "github.com/pbufio/pbuf-registry", + }, + args: args{ + outputPath: "api/v1", + content: withPackageProtoFile, + }, + want: withPackageProtoFilePatched, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := NewGoPackagePatcher(tt.fields.goModule) + + got, err := p.Patch(tt.args.outputPath, tt.args.content) + if (err != nil) != tt.wantErr { + t.Errorf("Patch() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Patch() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/patcher/patcher.go b/internal/patcher/patcher.go new file mode 100644 index 0000000..9d1a58f --- /dev/null +++ b/internal/patcher/patcher.go @@ -0,0 +1,16 @@ +package patcher + +type Patcher interface { + Patch(outputPath, content string) (string, error) +} + +func ApplyPatchers(patchers []Patcher, outputPath string, content string) (string, error) { + for _, patcher := range patchers { + var err error + content, err = patcher.Patch(outputPath, content) + if err != nil { + return "", err + } + } + return content, nil +} diff --git a/internal/registry/vendor.go b/internal/registry/vendor.go index 5c0a5f9..7f73c1e 100644 --- a/internal/registry/vendor.go +++ b/internal/registry/vendor.go @@ -10,11 +10,13 @@ import ( v1 "github.com/pbufio/pbuf-cli/gen/pbuf-registry/v1" "github.com/pbufio/pbuf-cli/internal/model" + "github.com/pbufio/pbuf-cli/internal/patcher" ) const timeout = 60 * time.Second -func VendorRegistryModule(module *model.Module, client v1.RegistryClient) error { +// VendorRegistryModule function that iterate over the modules and vendor proto files from PBUF registry +func VendorRegistryModule(module *model.Module, client v1.RegistryClient, patchers []patcher.Patcher) error { log.Printf("start vendoring .proto files. module name: %s, path: %s", module.Name, module.Path) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -61,7 +63,23 @@ func VendorRegistryModule(module *model.Module, client v1.RegistryClient) error } } - err := os.MkdirAll(filepath.Dir(originalFilename), os.ModePerm) + var content string + outputDir := filepath.Dir(originalFilename) + + if module.GenerateOutputFolder != "" { + content, err = patcher.ApplyPatchers( + patchers, + strings.Replace(outputDir, module.OutputFolder, module.GenerateOutputFolder, 1), + protoFile.Content, + ) + if err != nil { + log.Printf("failed to patch file %s: %v", originalFilename, err) + } + } else { + content = protoFile.Content + } + + err = os.MkdirAll(outputDir, os.ModePerm) if err != nil { log.Printf("failed to create directory: %s", outputPath) return err @@ -73,7 +91,7 @@ func VendorRegistryModule(module *model.Module, client v1.RegistryClient) error return err } - _, err = copiedFile.Write([]byte(protoFile.Content)) + _, err = copiedFile.Write([]byte(content)) if err != nil { log.Printf("failed to write file contents: %s", outputPath) return err diff --git a/pbuf.yaml b/pbuf.yaml index 0afc1a3..c1de6df 100644 --- a/pbuf.yaml +++ b/pbuf.yaml @@ -8,6 +8,7 @@ modules: tag: v0.2.0 path: api/pbuf-registry out: third_party/pbuf-registry + gen_out: gen/pbuf-registry - name: googleapis repository: https://github.com/googleapis/googleapis path: google/api diff --git a/third_party/pbuf-registry/v1/entities.proto b/third_party/pbuf-registry/v1/entities.proto index 5b19e46..7692a94 100644 --- a/third_party/pbuf-registry/v1/entities.proto +++ b/third_party/pbuf-registry/v1/entities.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package pbufregistry.v1; -option go_package = "pbufregistry/api/v1;v1"; +option go_package = "github.com/pbufio/pbuf-cli/gen/pbuf-registry/v1;v1"; // Module is a module registered in the registry. message Module { diff --git a/third_party/pbuf-registry/v1/registry.proto b/third_party/pbuf-registry/v1/registry.proto index f6753d0..c847c1c 100644 --- a/third_party/pbuf-registry/v1/registry.proto +++ b/third_party/pbuf-registry/v1/registry.proto @@ -5,7 +5,7 @@ package pbufregistry.v1; import "google/api/annotations.proto"; import "pbuf-registry/v1/entities.proto"; -option go_package = "pbufregistry/api/v1;v1"; +option go_package = "github.com/pbufio/pbuf-cli/gen/pbuf-registry/v1;v1"; // Registry service definition service Registry {