diff --git a/mmv1/api/product.go b/mmv1/api/product.go index 7539fa5eda9f..37e9dc932178 100644 --- a/mmv1/api/product.go +++ b/mmv1/api/product.go @@ -34,12 +34,12 @@ type Product struct { // original value of :name before the provider override happens // same as :name if not overridden in provider - ApiName string `yaml:"api_name"` + ApiName string `yaml:"api_name,omitempty"` // Display Name: The full name of the GCP product; eg "Cloud Bigtable" - DisplayName string `yaml:"display_name"` + DisplayName string `yaml:"display_name,omitempty"` - Objects []*Resource + Objects []*Resource `yaml:"objects,omitempty"` // The list of permission scopes available for the service // For example: `https://www.googleapis.com/auth/compute` @@ -50,19 +50,19 @@ type Product struct { // The base URL for the service API endpoint // For example: `https://www.googleapis.com/compute/v1/` - BaseUrl string `yaml:"base_url"` + BaseUrl string `yaml:"base_url,omitempty"` // A function reference designed for the rare case where you // need to use retries in operation calls. Used for the service api // as it enables itself (self referential) and can result in occasional // failures on operation_get. see github.com/hashicorp/terraform-provider-google/issues/9489 - OperationRetry string `yaml:"operation_retry"` + OperationRetry string `yaml:"operation_retry,omitempty"` - Async *Async + Async *Async `yaml:"async,omitempty"` - LegacyName string `yaml:"legacy_name"` + LegacyName string `yaml:"legacy_name,omitempty"` - ClientName string `yaml:"client_name"` + ClientName string `yaml:"client_name,omitempty"` } func (p *Product) UnmarshalYAML(unmarshal func(any) error) error { diff --git a/mmv1/api/product/version.go b/mmv1/api/product/version.go index 16ef54ae51b0..d94c0a41e8e5 100644 --- a/mmv1/api/product/version.go +++ b/mmv1/api/product/version.go @@ -26,7 +26,7 @@ var ORDER = []string{"ga", "beta", "alpha", "private"} // a superset of beta, and beta a superset of GA. Each version will have a // different version url. type Version struct { - CaiBaseUrl string `yaml:"cai_base_url"` + CaiBaseUrl string `yaml:"cai_base_url,omitempty"` BaseUrl string `yaml:"base_url"` Name string } diff --git a/mmv1/go.mod b/mmv1/go.mod index 79316b0a09ea..839dd55d3c5d 100644 --- a/mmv1/go.mod +++ b/mmv1/go.mod @@ -11,4 +11,15 @@ require github.com/golang/glog v1.2.0 require github.com/otiai10/copy v1.9.0 -require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect +require ( + github.com/getkin/kin-openapi v0.127.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/invopop/yaml v0.3.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/mmv1/go.sum b/mmv1/go.sum index 14b659976106..8d1e3cdeb383 100644 --- a/mmv1/go.sum +++ b/mmv1/go.sum @@ -1,13 +1,29 @@ +github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= +github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= +github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= @@ -16,3 +32,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mmv1/main.go b/mmv1/main.go index 19ac6c6c7ec7..d71410cbc820 100644 --- a/mmv1/main.go +++ b/mmv1/main.go @@ -17,6 +17,7 @@ import ( "golang.org/x/exp/slices" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api" + "github.com/GoogleCloudPlatform/magic-modules/mmv1/openapi_generate" "github.com/GoogleCloudPlatform/magic-modules/mmv1/provider" ) @@ -42,6 +43,8 @@ var doNotGenerateDocs = flag.Bool("no-docs", false, "do not generate docs") var forceProvider = flag.String("provider", "", "optional provider name. If specified, a non-default provider will be used.") +var openapiGenerate = flag.Bool("openapi-generate", false, "Generate MMv1 YAML from openapi directory (Experimental)") + // Example usage: --yaml var yamlMode = flag.Bool("yaml", false, "copy text over from ruby yaml to go yaml") @@ -62,6 +65,12 @@ func main() { flag.Parse() + if *openapiGenerate { + parser := openapi_generate.NewOpenapiParser("openapi_generate/openapi", "products") + parser.Run() + return + } + if *yamlMode || *yamlTempMode { CopyAllDescriptions(*yamlTempMode) } diff --git a/mmv1/openapi_generate/parser.go b/mmv1/openapi_generate/parser.go new file mode 100644 index 000000000000..3dbaa1edf634 --- /dev/null +++ b/mmv1/openapi_generate/parser.go @@ -0,0 +1,324 @@ +// Copyright 2024 Google Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generator for a library converting terraform state to gcp objects. + +package openapi_generate + +import ( + "bytes" + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "log" + + "text/template" + + "github.com/GoogleCloudPlatform/magic-modules/mmv1/api" + "github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product" + "github.com/GoogleCloudPlatform/magic-modules/mmv1/google" + "github.com/getkin/kin-openapi/openapi3" + "github.com/golang/glog" + "gopkg.in/yaml.v2" +) + +type Parser struct { + Folder string + Output string +} + +func NewOpenapiParser(folder, output string) Parser { + + wd, err := os.Getwd() + if err != nil { + log.Fatalf(err.Error()) + } + + parser := Parser{ + Folder: path.Join(wd, folder), + Output: path.Join(wd, output), + } + + return parser +} + +func (parser Parser) Run() { + + f, err := os.Open(parser.Folder) + if err != nil { + log.Fatalf(err.Error()) + return + } + defer f.Close() + files, err := f.Readdirnames(0) + if err != nil { + log.Fatalf(err.Error()) + } + + // check if folder is empty + if len(files) == 0 { + log.Fatalf("No OpenAPI files found in %s", parser.Folder) + } + + for _, file := range files { + parser.WriteYaml(path.Join(parser.Folder, file)) + } +} + +func (parser Parser) WriteYaml(filePath string) { + log.Printf("Reading from file path %s", filePath) + + ctx := context.Background() + loader := &openapi3.Loader{Context: ctx, IsExternalRefsAllowed: true} + doc, _ := loader.LoadFromFile(filePath) + _ = doc.Validate(ctx) + + resourcePaths := findResources(doc) + productPath := buildProduct(filePath, parser.Output, doc) + + log.Printf("Generated product %+v/product.yaml", productPath) + for _, pathArray := range resourcePaths { + resource := buildResource(filePath, pathArray[0], pathArray[1], doc) + + // template method + resourceOutPathTemplate := filepath.Join(productPath, fmt.Sprintf("%s_template.yaml", resource.Name)) + templatePath := "openapi_generate/resource_yaml.tmpl" + WriteGoTemplate(templatePath, resourceOutPathTemplate, resource) + log.Printf("Generated resource %s", resourceOutPathTemplate) + + // marshal method + resourceOutPathMarshal := filepath.Join(productPath, fmt.Sprintf("%s_marshal.yaml", resource.Name)) + bytes, err := yaml.Marshal(resource) + if err != nil { + log.Fatalf("error marshalling yaml %v: %v", resourceOutPathMarshal, err) + } + err = os.WriteFile(resourceOutPathMarshal, bytes, 0644) + if err != nil { + log.Fatalf("error writing product to path %v: %v", resourceOutPathMarshal, err) + } + log.Printf("Generated resource %s", resourceOutPathMarshal) + } +} + +func findResources(doc *openapi3.T) [][]string { + var resourcePaths [][]string + + pathMap := doc.Paths.Map() + for key, pathValue := range pathMap { + if pathValue.Post == nil { + continue + } + + // Not very clever way of identifying create resource methods + if strings.HasPrefix(pathValue.Post.OperationID, "Create") { + resourcePath := key + resourceName := strings.Replace(pathValue.Post.OperationID, "Create", "", 1) + resourcePaths = append(resourcePaths, []string{resourcePath, resourceName}) + } + } + + return resourcePaths +} + +func buildProduct(filePath, output string, root *openapi3.T) string { + + version := root.Info.Version + server := root.Servers[0].URL + + productName := strings.Split(filepath.Base(filePath), "_")[0] + productPath := filepath.Join(output, productName) + + if err := os.MkdirAll(productPath, os.ModePerm); err != nil { + log.Fatalf("error creating product output directory %v: %v", productPath, err) + } + + apiProduct := &api.Product{} + apiVersion := &product.Version{} + + apiVersion.BaseUrl = fmt.Sprintf("%s/%s/", server, version) + // TODO(slevenick) figure out how to tell the API version + apiVersion.Name = "ga" + apiProduct.Versions = []*product.Version{apiVersion} + + // Standard titling is "Service Name API" + displayName := strings.Replace(root.Info.Title, " API", "", 1) + apiProduct.Name = strings.ReplaceAll(displayName, " ", "") + apiProduct.DisplayName = displayName + + //Scopes should be added soon to OpenAPI, until then use global scope + apiProduct.Scopes = []string{"https://www.googleapis.com/auth/cloud-platform"} + + // productOutPath := filepath.Join(output, fmt.Sprintf("/%s/product.yaml", productName)) + templatePath := "openapi_generate/product_yaml.tmpl" + + productOutPathTemplate := filepath.Join(output, fmt.Sprintf("/%s/product_template.yaml", productName)) + WriteGoTemplate(templatePath, productOutPathTemplate, apiProduct) + + productOutPathMarshal := filepath.Join(output, fmt.Sprintf("/%s/product_marshal.yaml", productName)) + + // Default yaml marshaller + bytes, err := yaml.Marshal(apiProduct) + if err != nil { + log.Fatalf("error marshalling yaml %v: %v", productOutPathMarshal, err) + } + + err = os.WriteFile(productOutPathMarshal, bytes, 0644) + if err != nil { + log.Fatalf("error writing product to path %v: %v", productOutPathMarshal, err) + } + + return productPath +} + +func buildResource(filePath, resourcePath, resourceName string, root *openapi3.T) api.Resource { + resource := api.Resource{} + + parsedObjects := parseOpenApi(resourcePath, resourceName, root) + + parameters := parsedObjects[0].([]*api.Type) + properties := parsedObjects[1].([]*api.Type) + queryParam := parsedObjects[2].(string) + + // TODO base_url(resource_path) + baseUrl := resourcePath + selfLink := fmt.Sprintf("%s/%s", baseUrl, strings.ToLower(queryParam)) + + resource.Name = resourceName + resource.Parameters = parameters + resource.Properties = properties + resource.SelfLink = selfLink + + return resource +} + +func parseOpenApi(resourcePath, resourceName string, root *openapi3.T) []any { + returnArray := []any{} + path := root.Paths.Find(resourcePath) + + parameters := []*api.Type{} + var idParam string + for _, param := range path.Post.Parameters { + if strings.Contains(strings.ToLower(param.Value.Name), strings.ToLower(resourceName)) { + idParam = param.Value.Name + } + paramObj := writeObject(param.Value.Name, param.Value.Schema, *param.Value.Schema.Value.Type, true) + + if param.Value.Name == "requestId" || param.Value.Name == "validateOnly" || paramObj.Name == "" { + continue + } + + // All parameters are immutable + paramObj.Immutable = true + parameters = append(parameters, ¶mObj) + } + + // TODO build_properties + properties := []*api.Type{} + + returnArray = append(returnArray, parameters) + returnArray = append(returnArray, properties) + returnArray = append(returnArray, idParam) + + return returnArray +} + +func writeObject(name string, obj *openapi3.SchemaRef, objType openapi3.Types, urlParam bool) api.Type { + var field api.Type + + switch name { + case "projectsId", "project": + // projectsId and project are omitted in MMv1 as they are inferred from + // the presence of {{project}} in the URL + return field + case "locationsId": + name = "location" + } + additionalDescription := "" + + // log.Printf("%s %+v", name, obj.Value.AllOf) + + if len(obj.Value.AllOf) > 0 { + obj = obj.Value.AllOf[0] + objType = *obj.Value.Type + } + + if objType.Is("string") { + field.Type = "string" + field.Name = name + if len(obj.Value.Enum) > 0 { + var enums []string + for _, enum := range obj.Value.Enum { + enums = append(enums, fmt.Sprintf("%v", enum)) + } + additionalDescription = fmt.Sprintf("\n Possible values:\n %s", strings.Join(enums, "\n")) + } + } + + description := fmt.Sprintf("%s %s", obj.Value.Description, additionalDescription) + if strings.TrimSpace(description) == "" { + description = "No description" + } + + if urlParam { + field.UrlParamOnly = true + field.Required = true + } + + // These methods are only available when the field is set + if obj.Value.ReadOnly { + field.Output = true + } + + // x-google-identifier fields are described by AIP 203 and are represented + // as output only in Terraform. + xGoogleId, err := obj.JSONLookup("x-google-identifier") + if err == nil && xGoogleId != nil { + field.Output = true + } + + xGoogleImmutable, err := obj.JSONLookup("x-google-immutable") + if obj.Value.ReadOnly || (err == nil && xGoogleImmutable != nil) { + field.Immutable = true + } + + return field +} + +func WriteGoTemplate(templatePath, filePath string, input any) { + contents := bytes.Buffer{} + + templateFileName := filepath.Base(templatePath) + templates := []string{ + templatePath, + } + + tmpl, err := template.New(templateFileName).Funcs(google.TemplateFunctions).ParseFiles(templates...) + if err != nil { + glog.Exit(fmt.Sprintf("error parsing %s for filepath %s ", templatePath, filePath), err) + } + if err = tmpl.ExecuteTemplate(&contents, templateFileName, input); err != nil { + glog.Exit(fmt.Sprintf("error executing %s for filepath %s ", templatePath, filePath), err) + } + + bytes := contents.Bytes() + + err = os.WriteFile(filePath, bytes, 0644) + if err != nil { + log.Fatalf("error writing product to path %v: %v", filePath, err) + } + +} diff --git a/mmv1/openapi_generate/product_yaml.tmpl b/mmv1/openapi_generate/product_yaml.tmpl new file mode 100644 index 000000000000..38bd016db19c --- /dev/null +++ b/mmv1/openapi_generate/product_yaml.tmpl @@ -0,0 +1,25 @@ +# Copyright 2024 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: '{{$.Name}}' +display_name: '{{$.DisplayName}}' +versions: +{{- range $version := $.Versions }} + - name: '{{$version.Name}}' + base_url: {{$version.BaseUrl}}' +{{- end }} +scopes: +{{- range $scope := $.Scopes }} + - '{{$scope}}' +{{- end }} diff --git a/mmv1/openapi_generate/resource_yaml.tmpl b/mmv1/openapi_generate/resource_yaml.tmpl new file mode 100644 index 000000000000..d6182b0a6233 --- /dev/null +++ b/mmv1/openapi_generate/resource_yaml.tmpl @@ -0,0 +1,15 @@ +# Copyright 2024 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: '{{$.Name}}'