From ce5406552e952adf75f3e57b43edef17ff4e964c Mon Sep 17 00:00:00 2001 From: justinsb Date: Mon, 24 Jun 2024 21:27:05 -0400 Subject: [PATCH] cli: add ability to generate types for CRD Example usage: generate-types --proto-source-path ../proto-to-mapper/build/googleapis.pb \ --service google.cloud.kms.v1 --version v1beta1 --output-api ~/kcc/k8s-config-connector/ --- dev/tools/controllerbuilder/cmd/root.go | 36 ++- dev/tools/controllerbuilder/go.mod | 7 +- dev/tools/controllerbuilder/go.sum | 41 +++ .../pkg/codegen/generatorbase.go | 83 ++++++ .../pkg/codegen/typegenerator.go | 282 ++++++++++++++++++ .../generatetypes/generatetypescommand.go | 114 +++++++ .../pkg/options/generateoptions.go | 34 +++ .../controllerbuilder/pkg/protoapi/loader.go | 64 ++++ dev/tools/proto-to-mapper/Makefile | 6 +- 9 files changed, 650 insertions(+), 17 deletions(-) create mode 100644 dev/tools/controllerbuilder/pkg/codegen/generatorbase.go create mode 100644 dev/tools/controllerbuilder/pkg/codegen/typegenerator.go create mode 100644 dev/tools/controllerbuilder/pkg/commands/generatetypes/generatetypescommand.go create mode 100644 dev/tools/controllerbuilder/pkg/options/generateoptions.go create mode 100644 dev/tools/controllerbuilder/pkg/protoapi/loader.go diff --git a/dev/tools/controllerbuilder/cmd/root.go b/dev/tools/controllerbuilder/cmd/root.go index fde64c65c2..8bb3703819 100644 --- a/dev/tools/controllerbuilder/cmd/root.go +++ b/dev/tools/controllerbuilder/cmd/root.go @@ -19,46 +19,52 @@ import ( "os" "strings" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/commands/generatetypes" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/options" "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/scaffold" "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/template" "github.com/spf13/cobra" ) -var ( - serviceName string +func buildAddCommand(baseOptions *options.GenerateOptions) *cobra.Command { // TODO: Resource and kind name should be the same. Validation the uppercase/lowercase. - kind string - apiVersion string + kind := "" - addCmd = &cobra.Command{ + addCmd := &cobra.Command{ Use: "add", Short: "add direct controller", RunE: func(cmd *cobra.Command, args []string) error { // TODO(check kcc root) cArgs := &template.ControllerArgs{ - Service: serviceName, - Version: apiVersion, + Service: baseOptions.ServiceName, + Version: baseOptions.APIVersion, Kind: kind, KindToLower: strings.ToLower(kind), } - path, err := scaffold.BuildControllerPath(serviceName, kind) + path, err := scaffold.BuildControllerPath(baseOptions.ServiceName, kind) if err != nil { return err } return scaffold.Scaffold(path, cArgs) }, } -) - -func init() { - addCmd.PersistentFlags().StringVarP(&apiVersion, "version", "v", "v1alpha1", "the KRM API version. used to import the KRM API") - addCmd.PersistentFlags().StringVarP(&serviceName, "service", "s", "", "the GCP service name") addCmd.PersistentFlags().StringVarP(&kind, "resourceInKind", "r", "", "the GCP resource name under the GCP service. should be in camel case ") + + return addCmd } func Execute() { - if err := addCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) + var generateOptions options.GenerateOptions + generateOptions.InitDefaults() + + rootCmd := &cobra.Command{} + generateOptions.BindPersistentFlags(rootCmd) + + rootCmd.AddCommand(buildAddCommand(&generateOptions)) + rootCmd.AddCommand(generatetypes.BuildCommand(&generateOptions)) + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } } diff --git a/dev/tools/controllerbuilder/go.mod b/dev/tools/controllerbuilder/go.mod index 295045f821..d2064fc6d3 100644 --- a/dev/tools/controllerbuilder/go.mod +++ b/dev/tools/controllerbuilder/go.mod @@ -1,6 +1,6 @@ module github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder -go 1.22 +go 1.22.0 toolchain go1.22.5 @@ -8,9 +8,14 @@ require ( github.com/fatih/color v1.17.0 github.com/spf13/cobra v1.8.0 golang.org/x/tools v0.21.0 + google.golang.org/protobuf v1.34.2 + k8s.io/apimachinery v0.30.2 + k8s.io/klog/v2 v2.130.1 ) require ( + github.com/go-logr/logr v1.4.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/dev/tools/controllerbuilder/go.sum b/dev/tools/controllerbuilder/go.sum index 8aa644a86b..d32910d571 100644 --- a/dev/tools/controllerbuilder/go.sum +++ b/dev/tools/controllerbuilder/go.sum @@ -1,8 +1,16 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -13,15 +21,48 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= diff --git a/dev/tools/controllerbuilder/pkg/codegen/generatorbase.go b/dev/tools/controllerbuilder/pkg/codegen/generatorbase.go new file mode 100644 index 0000000000..e990fb3474 --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/codegen/generatorbase.go @@ -0,0 +1,83 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codegen + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" +) + +type generatorBase struct { + generatedFiles map[generatedFileKey]*generatedFile +} + +func (g *generatorBase) init() { + g.generatedFiles = make(map[generatedFileKey]*generatedFile) +} + +func (g *generatorBase) getOutputFile(k generatedFileKey) *generatedFile { + out := g.generatedFiles[k] + if out == nil { + out = &generatedFile{key: k} + g.generatedFiles[k] = out + } + return out +} + +type generatedFile struct { + key generatedFileKey + contents bytes.Buffer +} + +type generatedFileKey struct { + GoPackagePath string + + File string +} + +func (f *generatedFile) Write(baseDir string) error { + if f.contents.Len() == 0 { + return nil + } + + fullName := f.key.GoPackagePath + tokens := strings.Split(fullName, ".") + dirTokens := []string{baseDir} + dirTokens = append(dirTokens, tokens...) + dir := filepath.Join(dirTokens...) + + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating directory %q: %w", dir, err) + } + + p := filepath.Join(dir, f.key.File) + if err := os.WriteFile(p, f.contents.Bytes(), 0644); err != nil { + return fmt.Errorf("writing %q: %w", p, err) + } + + return nil +} + +func (v *generatorBase) WriteFiles(baseDir string) error { + for _, f := range v.generatedFiles { + if err := f.Write(baseDir); err != nil { + return err + } + } + return nil +} diff --git a/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go b/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go new file mode 100644 index 0000000000..525f5a9afb --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/codegen/typegenerator.go @@ -0,0 +1,282 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codegen + +import ( + "fmt" + "io" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + protoapi "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/protoapi" + + "google.golang.org/protobuf/reflect/protoreflect" + "k8s.io/klog/v2" +) + +type TypeGenerator struct { + generatorBase + goPathForMessage OutputFunc +} + +func NewTypeGenerator(goPathForMessage OutputFunc) *TypeGenerator { + g := &TypeGenerator{ + goPathForMessage: goPathForMessage, + } + g.generatorBase.init() + return g +} + +type OutputFunc func(msg protoreflect.MessageDescriptor) (goPath string, shouldWrite bool) + +func (v *TypeGenerator) VisitProto(api *protoapi.Proto) error { + sortedFiles := api.SortedFiles() + for _, f := range sortedFiles { + v.visitFile(f) + } + return nil +} + +func (g *TypeGenerator) visitFile(f protoreflect.FileDescriptor) { + for _, msg := range sorted(f.Messages()) { + g.visitMessage(msg) + } + + { + + for _, msg := range sorted(f.Messages()) { + if msg.IsMapEntry() { + continue + } + + goPath, ok := g.goPathForMessage(msg) + if !ok { + continue + } + + krmVersion := filepath.Base(goPath) + + k := generatedFileKey{ + GoPackagePath: goPath, + File: "types.generated.go", + } + out := g.getOutputFile(k) + + w := &out.contents + + if out.contents.Len() == 0 { + writeCopyright(w, time.Now().Year()) + + fmt.Fprintf(w, "package %s\n", krmVersion) + } + + g.writeTypes(w, msg) + } + } + +} + +func writeCopyright(w io.Writer, year int) { + s := `// Copyright {{.Year}} Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +` + s = strings.ReplaceAll(s, "{{.Year}}", strconv.Itoa(year)) + if _, err := w.Write([]byte(s)); err != nil { + klog.Fatalf("writing copyright: %v", err) + } +} + +func (v *TypeGenerator) visitMessage(msg protoreflect.MessageDescriptor) { + if _, visit := v.goPathForMessage(msg); !visit { + return + } + // goTypes := v.findKRMStructsForProto(msg) + + // if len(goTypes) == 0 { + // klog.V(2).Infof("no krm for %v", msg.FullName()) + // return + // } + // for _, goType := range goTypes { + // v.typePairs = append(v.typePairs, typePair{ + // ProtoPackage: msg.ParentFile().Package(), + // KRMType: goType, + // Proto: msg, + // }) + // } + + for _, msg := range sorted(msg.Messages()) { + v.visitMessage(msg) + } +} + +func (v *TypeGenerator) writeTypes(out io.Writer, msg protoreflect.MessageDescriptor) { + goType := goNameForProtoMessage(msg, msg) + + { + fmt.Fprintf(out, "// +kcc:proto=%s\n", msg.FullName()) + fmt.Fprintf(out, "type %s struct {\n", goType) + for i := 0; i < msg.Fields().Len(); i++ { + field := msg.Fields().Get(i) + sourceLocations := msg.ParentFile().SourceLocations().ByDescriptor(field) + + goFieldName := strings.Title(field.JSONName()) + jsonName := field.JSONName() + goType := "" + + if field.IsMap() { + entryMsg := field.Message() + keyKind := entryMsg.Fields().ByName("key").Kind() + valueKind := entryMsg.Fields().ByName("value").Kind() + if keyKind == protoreflect.StringKind && valueKind == protoreflect.StringKind { + goType = "map[string]string" + } else if keyKind == protoreflect.StringKind && valueKind == protoreflect.Int64Kind { + goType = "map[string]int64" + } else { + fmt.Fprintf(out, "// TODO: map type %v %v\n", keyKind, valueKind) + } + } else { + switch field.Kind() { + case protoreflect.MessageKind: + goType = goNameForProtoMessage(msg, field.Message()) + + case protoreflect.EnumKind: + goType = "string" //string(field.Enum().Name()) + + default: + goType = goTypeForProtoKind(field.Kind()) + } + + if field.Cardinality() == protoreflect.Repeated { + goType = "[]" + goType + } else { + goType = "*" + goType + } + } + + // Blank line between fields for readability + if i != 0 { + fmt.Fprintf(out, "\n") + } + + if sourceLocations.LeadingComments != "" { + comment := strings.TrimSpace(sourceLocations.LeadingComments) + for _, line := range strings.Split(comment, "\n") { + fmt.Fprintf(out, " // %s\n", line) + } + } + + fmt.Fprintf(out, " %s %s `json:\"%s,omitempty\"`\n", + goFieldName, + goType, + jsonName, + ) + } + fmt.Fprintf(out, "}\n") + } + + for i := 0; i < msg.Messages().Len(); i++ { + m := msg.Messages().Get(i) + if m.IsMapEntry() { + continue + } + v.writeTypes(out, m) + } + +} + +func sorted(messages protoreflect.MessageDescriptors) []protoreflect.MessageDescriptor { + var out []protoreflect.MessageDescriptor + for i := 0; i < messages.Len(); i++ { + m := messages.Get(i) + out = append(out, m) + } + sort.Slice(out, func(i, j int) bool { + return out[i].FullName() < out[j].FullName() + }) + return out +} + +func goNameForProtoMessage(parentMessage protoreflect.MessageDescriptor, msg protoreflect.MessageDescriptor) string { + fullName := string(msg.FullName()) + fullName = strings.TrimPrefix(fullName, string(parentMessage.ParentFile().FullName())) + fullName = strings.TrimPrefix(fullName, ".") + fullName = strings.ReplaceAll(fullName, ".", "_") + + // Some special-case values that are not obvious how to map in KRM + switch fullName { + case "google_protobuf_Timestamp": + return "string" + case "google_protobuf_Duration": + return "string" + case "google_protobuf_Int64Value": + return "int64" + } + return fullName +} + +func goTypeForProtoKind(kind protoreflect.Kind) string { + goType := "" + switch kind { + case protoreflect.StringKind: + goType = "string" + + case protoreflect.Int32Kind: + goType = "int32" + + case protoreflect.Int64Kind: + goType = "int64" + + case protoreflect.Uint32Kind: + goType = "uint32" + + case protoreflect.Uint64Kind: + goType = "uint64" + + case protoreflect.Fixed64Kind: + goType = "uint64" + + case protoreflect.BoolKind: + goType = "bool" + + case protoreflect.DoubleKind: + goType = "float64" + + case protoreflect.FloatKind: + goType = "float32" + + case protoreflect.BytesKind: + goType = "[]byte" + + default: + klog.Fatalf("unhandled kind %q", kind) + } + + return goType +} diff --git a/dev/tools/controllerbuilder/pkg/commands/generatetypes/generatetypescommand.go b/dev/tools/controllerbuilder/pkg/commands/generatetypes/generatetypescommand.go new file mode 100644 index 0000000000..b5a563e28b --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/commands/generatetypes/generatetypescommand.go @@ -0,0 +1,114 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generatetypes + +import ( + "context" + "fmt" + "strings" + + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/codegen" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/options" + "github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/protoapi" + "google.golang.org/protobuf/reflect/protoreflect" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/spf13/cobra" +) + +type GenerateCRDOptions struct { + *options.GenerateOptions + + OutputAPIDirectory string +} + +func (o *GenerateCRDOptions) BindFlags(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.OutputAPIDirectory, "output-api", o.OutputAPIDirectory, "base directory for writing APIs") +} + +func BuildCommand(baseOptions *options.GenerateOptions) *cobra.Command { + opt := &GenerateCRDOptions{ + GenerateOptions: baseOptions, + } + + cmd := &cobra.Command{ + Use: "generate-types", + Short: "generate KRM types for a proto service", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if err := RunGenerateCRD(ctx, opt); err != nil { + return err + } + return nil + }, + } + + opt.BindFlags(cmd) + + return cmd +} + +func RunGenerateCRD(ctx context.Context, o *GenerateCRDOptions) error { + if o.ServiceName == "" { + return fmt.Errorf("ServiceName is required") + } + if o.GenerateOptions.ProtoSourcePath == "" { + return fmt.Errorf("ProtoSourcePath is required") + } + + gv, err := schema.ParseGroupVersion(o.APIVersion) + if err != nil { + return fmt.Errorf("APIVersion %q is not valid: %w", o.APIVersion, err) + } + + api, err := protoapi.LoadProto(o.GenerateOptions.ProtoSourcePath) + if err != nil { + return fmt.Errorf("loading proto: %w", err) + } + + pathForMessage := func(msg protoreflect.MessageDescriptor) (string, bool) { + fullName := string(msg.FullName()) + if strings.HasSuffix(fullName, "Request") { + return "", false + } + if strings.HasSuffix(fullName, "Response") { + return "", false + } + + if !strings.HasPrefix(fullName, o.ServiceName) { + return "", false + } + + protoPackagePath := string(msg.ParentFile().Package()) + protoPackagePath = strings.TrimPrefix(protoPackagePath, "google.") + protoPackagePath = strings.TrimPrefix(protoPackagePath, "cloud.") + protoPackagePath = strings.TrimSuffix(protoPackagePath, ".v1") + protoPackagePath = strings.TrimSuffix(protoPackagePath, ".v1beta1") + goPackage := "apis/" + strings.Join(strings.Split(protoPackagePath, "."), "/") + "/" + gv.Version + + return goPackage, true + } + typeGenerator := codegen.NewTypeGenerator(pathForMessage) + if err := typeGenerator.VisitProto(api); err != nil { + return err + } + + if err := typeGenerator.WriteFiles(o.OutputAPIDirectory); err != nil { + return err + } + + return nil + +} diff --git a/dev/tools/controllerbuilder/pkg/options/generateoptions.go b/dev/tools/controllerbuilder/pkg/options/generateoptions.go new file mode 100644 index 0000000000..88d6e3026b --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/options/generateoptions.go @@ -0,0 +1,34 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import "github.com/spf13/cobra" + +type GenerateOptions struct { + ProtoSourcePath string + ServiceName string + APIVersion string +} + +func (o *GenerateOptions) InitDefaults() { + o.APIVersion = "v1alpha1" +} + +func (o *GenerateOptions) BindPersistentFlags(cmd *cobra.Command) { + cmd.PersistentFlags().StringVar(&o.ProtoSourcePath, "proto-source-path", o.ProtoSourcePath, "path to (compiled) proto for APIs") + cmd.PersistentFlags().StringVarP(&o.APIVersion, "version", "v", o.APIVersion, "the KRM API version. used to import the KRM API") + cmd.PersistentFlags().StringVarP(&o.ServiceName, "service", "s", o.ServiceName, "the GCP service name") + +} diff --git a/dev/tools/controllerbuilder/pkg/protoapi/loader.go b/dev/tools/controllerbuilder/pkg/protoapi/loader.go new file mode 100644 index 0000000000..b710713cdf --- /dev/null +++ b/dev/tools/controllerbuilder/pkg/protoapi/loader.go @@ -0,0 +1,64 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package protoapi + +import ( + "fmt" + "os" + "sort" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/descriptorpb" +) + +type Proto struct { + files *protoregistry.Files +} + +func LoadProto(p string) (*Proto, error) { + b, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", p, err) + } + + fds := &descriptorpb.FileDescriptorSet{} + if err := proto.Unmarshal(b, fds); err != nil { + return nil, fmt.Errorf("unmarshalling %q: %w", p, err) + } + + files, err := protodesc.NewFiles(fds) + if err != nil { + return nil, fmt.Errorf("building file description: %w", err) + } + + return &Proto{ + files: files, + }, nil +} + +func (p *Proto) SortedFiles() []protoreflect.FileDescriptor { + var sortedFiles []protoreflect.FileDescriptor + p.files.RangeFiles(func(f protoreflect.FileDescriptor) bool { + sortedFiles = append(sortedFiles, f) + return true + }) + sort.Slice(sortedFiles, func(i, j int) bool { + return sortedFiles[i].FullName() < sortedFiles[j].FullName() + }) + return sortedFiles +} diff --git a/dev/tools/proto-to-mapper/Makefile b/dev/tools/proto-to-mapper/Makefile index aa2829341b..9ecd85cb48 100644 --- a/dev/tools/proto-to-mapper/Makefile +++ b/dev/tools/proto-to-mapper/Makefile @@ -1,3 +1,7 @@ +.PHONY: generate-code +generate-code: generate-pb + go run . -apis ../../../apis/ --api-packages github.com/GoogleCloudPlatform/apis + .PHONY: generate-pb generate-pb: install-protoc-linux mkdir -p third_party @@ -10,10 +14,10 @@ generate-pb: install-protoc-linux ./third_party/googleapis/google/cloud/*/*/*.proto \ ./third_party/googleapis/google/iam/v1/*.proto \ ./third_party/googleapis/google/logging/v2/*.proto \ + ./third_party/googleapis/google/monitoring/v3/*.proto \ ./third_party/googleapis/google/monitoring/dashboard/v1/*.proto \ ./third_party/googleapis/google/devtools/cloudbuild/*/*.proto \ -o build/googleapis.pb - go run . -apis ../../../apis/ --api-packages github.com/GoogleCloudPlatform/apis install-protoc-linux: which protoc || sudo apt install -y protobuf-compiler