Skip to content

Commit

Permalink
Add option patcher to smooth compilation (#7)
Browse files Browse the repository at this point in the history
  • Loading branch information
aatarasoff authored Nov 16, 2023
1 parent cb9ff02 commit 945b81c
Show file tree
Hide file tree
Showing 14 changed files with 267 additions and 16 deletions.
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ docker:
lint:
golangci-lint run -v --timeout 10m

.PHONE: test
# test
test:
go test ./... -cover

# show help
help:
@echo ''
Expand Down
12 changes: 11 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -167,12 +167,14 @@ modules:
path: [path_in_registry]
tag: [dependency_module_tag]
out: [output_folder_on_local]
gen_out: [gen_output_folder_on_local] # optional, if provided then patchers will be applied
# use a git repository to vendor .proto files
- repository: [repository_url]
path: [path_in_repository]
branch: [branch_name]
tag: [tag_name]
out: [output_folder_on_local]
gen_out: [gen_output_folder_on_local] # optional, if provided then patchers will be applied
```
Replace main placeholders with appropriate values:
Expand All @@ -185,13 +187,15 @@ Replace placeholders in the registry modules with appropriate values:
- `[path_in_registry]`: Path to the folder or file in the registry you want to vendor.
- `[tag_name]`: Specific tag to vendor.
- `[output_folder_on_local]`: Folder where the vendor content should be placed on your local machine.
- `[gen_output_folder_on_local]`: Folder where the generated content should be placed on your local machine. Used to patch `go_package` option

Replace placeholders in modules placed in git with appropriate values:
- `[repository_url]`: The URL of the Git repository.
- `[path_in_repository]`: Path to the folder or file in the repository you want to vendor.
- `[branch_name]`: Specific branch name to clone (optional if tag is provided).
- `[tag_name]`: Specific tag to clone (optional if branch is provided).
- `[output_folder_on_local]`: Folder where the vendor content should be placed on your local machine.
- `[gen_output_folder_on_local]`: Folder where the generated content should be placed on your local machine. Used to patch `go_package` option

#### Examples

Expand Down Expand Up @@ -219,10 +223,16 @@ registry:
addr: pbuf.cloud:8081
insecure: true
modules:
# will copy api/v1/registry.proto file to third_party/api/v1/registry.proto
# will copy api/v1/*.proto file to third_party/api/v1/*.proto
- name: pbufio/pbuf-registry
tag: v0.0.1
out: third_party
# will copy api/v1/*.proto file to third_party/api/v1/*.proto
# and add or change `go_package` option to `<go_mod_name>/gen/pbuf-registry/api/v1`
- name: pbufio/pbuf-registry
tag: v0.0.1
out: third_party
gen_out: gen
# will copy examples/addressbook.proto file to proto/addressbook.proto
- repository: https://github.com/protocolbuffers/protobuf
path: examples/addressbook.proto
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/go-git/go-git/v5 v5.9.0
github.com/jdx/go-netrc v1.0.0
github.com/spf13/cobra v1.7.0
github.com/yoheimuta/go-protoparser/v4 v4.9.0
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.31.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/yoheimuta/go-protoparser/v4 v4.9.0 h1:zHRXzRjkOamwMkPu7bpiCtOpxHkM9c8zxQOvW99eWlo=
github.com/yoheimuta/go-protoparser/v4 v4.9.0/go.mod h1:AHNNnSWnb0UoL4QgHPiOAg2BniQceFscPI5X/BZNHl8=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
Expand Down
25 changes: 23 additions & 2 deletions internal/git/vendor.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@ import (
"github.com/go-git/go-git/v5/storage/memory"
"github.com/jdx/go-netrc"
"github.com/pbufio/pbuf-cli/internal/model"
"github.com/pbufio/pbuf-cli/internal/patcher"
)

func VendorGitModule(module *model.Module, netrcAuth *netrc.Netrc) error {
func VendorGitModule(module *model.Module, netrcAuth *netrc.Netrc, patchers []patcher.Patcher) error {
log.Printf("start vendoring .proto files. repo: %s, path: %s", module.Repository, module.Path)

var reference plumbing.ReferenceName
Expand Down Expand Up @@ -118,13 +119,33 @@ func VendorGitModule(module *model.Module, netrcAuth *netrc.Netrc) error {
return err
}

var content string
outputDir := filepath.Dir(outputPath)

if module.GenerateOutputFolder != "" {
content, err = patcher.ApplyPatchers(
patchers,
strings.Replace(outputDir, module.OutputFolder, module.GenerateOutputFolder, 1),
string(fileContents),
)
if err != nil {
log.Printf("failed to patch file %s: %v", outputPath, err)
}
} else {
content = string(fileContents)
}

if err != nil {
log.Printf("failed to patch file %s: %v", outputPath, err)
}

copiedFile, err := os.Create(outputPath)
if err != nil {
log.Printf("failed to create file: %s", outputPath)
return err
}

_, err = copiedFile.Write(fileContents)
_, err = copiedFile.Write([]byte(content))
if err != nil {
log.Printf("failed to write file contents: %s", outputPath)
return err
Expand Down
13 changes: 7 additions & 6 deletions internal/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@ type Config struct {
}

type Module struct {
Name string `yaml:"name"`
Repository string `yaml:"repository"`
Path string `yaml:"path"`
Branch string `yaml:"branch"`
Tag string `yaml:"tag"`
OutputFolder string `yaml:"out"`
Name string `yaml:"name"`
Repository string `yaml:"repository"`
Path string `yaml:"path"`
Branch string `yaml:"branch"`
Tag string `yaml:"tag"`
OutputFolder string `yaml:"out"`
GenerateOutputFolder string `yaml:"gen_out"`
}

func (c *Config) HasRegistry() bool {
Expand Down
27 changes: 25 additions & 2 deletions internal/modules/modules.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@ package modules

import (
"log"
"os"
"os/user"
"path/filepath"

"github.com/jdx/go-netrc"
v1 "github.com/pbufio/pbuf-cli/gen/pbuf-registry/v1"
"github.com/pbufio/pbuf-cli/internal/git"
"github.com/pbufio/pbuf-cli/internal/model"
"github.com/pbufio/pbuf-cli/internal/patcher"
"github.com/pbufio/pbuf-cli/internal/registry"
"golang.org/x/mod/modfile"
"gopkg.in/yaml.v2"
)

Expand All @@ -23,6 +26,24 @@ func NewConfig(contents []byte) (*model.Config, error) {
return modulesConfig, nil
}

// newProtoPatchers create a slice of proto patchers
func newProtoPatchers() []patcher.Patcher {
var result []patcher.Patcher

// if we have go.mod file
// then parse it and fetch the module name
// and pass it to the go package patcher
file, err := os.ReadFile("go.mod")
if err == nil {
// that's ok, we cannot find go mod file
path := modfile.ModulePath(file)
if path != "" {
result = append(result, patcher.NewGoPackagePatcher(path))
}
}
return result
}

// Vendor function that iterate over the modules and vendor proto files from git repositories
func Vendor(config *model.Config, client v1.RegistryClient) error {
usr, err := user.Current()
Expand All @@ -36,6 +57,8 @@ func Vendor(config *model.Config, client v1.RegistryClient) error {
log.Printf("no .netrc file found. skipping auth")
}

patchers := newProtoPatchers()

for _, module := range config.Modules {
if module.Repository == "" {
if config.HasRegistry() {
Expand All @@ -47,15 +70,15 @@ func Vendor(config *model.Config, client v1.RegistryClient) error {
log.Fatalf("no module tag found for module: %v", module)
}

err := registry.VendorRegistryModule(module, client)
err := registry.VendorRegistryModule(module, client, patchers)
if err != nil {
log.Fatalf("failed to vendor module %s: %v", module.Name, err)
}
} else {
log.Fatalf("no repository found for module: %s", module.Name)
}
} else {
err := git.VendorGitModule(module, netrcAuth)
err := git.VendorGitModule(module, netrcAuth, patchers)
if err != nil {
log.Fatalf("failed to vendor module %s: %v", module.Repository, err)
}
Expand Down
52 changes: 52 additions & 0 deletions internal/patcher/gopackage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package patcher

import (
"fmt"
"strings"

"github.com/yoheimuta/go-protoparser/v4/interpret/unordered"
)
import "github.com/yoheimuta/go-protoparser/v4"

type GoPackagePatcher struct {
goModule string
}

func NewGoPackagePatcher(goModule string) *GoPackagePatcher {
return &GoPackagePatcher{
goModule: goModule,
}
}

func (p *GoPackagePatcher) Patch(outputPath, content string) (string, error) {
parsed, err := protoparser.Parse(strings.NewReader(content))
if err != nil {
return "", err
}

proto, err := unordered.InterpretProto(parsed)
if err != nil {
return "", err
}

dirs := strings.Split(outputPath, "/")
goPackage := fmt.Sprintf(`option go_package = "%s/%s;%s";`, p.goModule, outputPath, dirs[len(dirs)-1])

for _, option := range proto.ProtoBody.Options {
if option.OptionName == "go_package" {
// break by lines
// option.Meta.Pos.Line as the line to change
splitted := strings.Split(content, "\n")
splitted[option.Meta.Pos.Line-1] = goPackage
return strings.Join(splitted, "\n"), nil
}
}

// if no go_package option, add it
splitted := strings.Split(content, "\n")
// add the element after syntax line
line := proto.Syntax.Meta.LastPos.Line
splitted = append(splitted[:line], append([]string{goPackage}, splitted[line:]...)...)

return strings.Join(splitted, "\n"), nil
}
101 changes: 101 additions & 0 deletions internal/patcher/gopackage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package patcher

import "testing"

const (
noPackageProtoFile = `
syntax = "proto3";
package pbufregistry.v1;
// Module is a module registered in the registry.
message Module {
}
`
noPackageProtoFilePatched = `
syntax = "proto3";
option go_package = "github.com/pbufio/pbuf-registry/api/v1;v1";
package pbufregistry.v1;
// Module is a module registered in the registry.
message Module {
}
`

withPackageProtoFile = `
syntax = "proto3";
package pbufregistry.v1;
option go_package = "pbufregistry/api/v2;v2";
// Module is a module registered in the registry.
message Module {
}
`

withPackageProtoFilePatched = `
syntax = "proto3";
package pbufregistry.v1;
option go_package = "github.com/pbufio/pbuf-registry/api/v1;v1";
// Module is a module registered in the registry.
message Module {
}
`
)

func TestGoPackagePatcher_Patch(t *testing.T) {
type fields struct {
goModule string
}
type args struct {
outputPath string
content string
}
tests := []struct {
name string
fields fields
args args
want string
wantErr bool
}{
{
name: "no package",
fields: fields{
goModule: "github.com/pbufio/pbuf-registry",
},
args: args{
outputPath: "api/v1",
content: noPackageProtoFile,
},
want: noPackageProtoFilePatched,
wantErr: false,
},
{
name: "with package",
fields: fields{
goModule: "github.com/pbufio/pbuf-registry",
},
args: args{
outputPath: "api/v1",
content: withPackageProtoFile,
},
want: withPackageProtoFilePatched,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := NewGoPackagePatcher(tt.fields.goModule)

got, err := p.Patch(tt.args.outputPath, tt.args.content)
if (err != nil) != tt.wantErr {
t.Errorf("Patch() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("Patch() got = %v, want %v", got, tt.want)
}
})
}
}
16 changes: 16 additions & 0 deletions internal/patcher/patcher.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package patcher

type Patcher interface {
Patch(outputPath, content string) (string, error)
}

func ApplyPatchers(patchers []Patcher, outputPath string, content string) (string, error) {
for _, patcher := range patchers {
var err error
content, err = patcher.Patch(outputPath, content)
if err != nil {
return "", err
}
}
return content, nil
}
Loading

0 comments on commit 945b81c

Please sign in to comment.