From 82c2b776e8c9c61ef111291f5ac0cf7b75a09b49 Mon Sep 17 00:00:00 2001 From: Scott Ganyo Date: Thu, 22 Jun 2023 16:47:14 -0700 Subject: [PATCH 1/3] fix conformance for yaml archives (breaks others) --- .../cmd/compute/conformance/conformance.go | 2 +- cmd/registry/patch/artifact.go | 28 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cmd/registry/cmd/compute/conformance/conformance.go b/cmd/registry/cmd/compute/conformance/conformance.go index 1ae274b95..0b087c317 100644 --- a/cmd/registry/cmd/compute/conformance/conformance.go +++ b/cmd/registry/cmd/compute/conformance/conformance.go @@ -31,7 +31,7 @@ import ( "github.com/spf13/cobra" ) -var styleguideFilter = fmt.Sprintf("mime_type.contains('%s')", mime.MimeTypeForKind("StyleGuide")) +var styleguideFilter = fmt.Sprintf("mime_type.contains('%s') || mime_type.contains('%s')", mime.MimeTypeForKind("StyleGuide"), mime.YamlMimeTypeForKind("StyleGuide")) func Command() *cobra.Command { var filter string diff --git a/cmd/registry/patch/artifact.go b/cmd/registry/patch/artifact.go index 04efcd793..8f137e224 100644 --- a/cmd/registry/patch/artifact.go +++ b/cmd/registry/patch/artifact.go @@ -122,31 +122,39 @@ func applyArtifactPatch(ctx context.Context, client connection.RegistryClient, c if err != nil { return err } - // Populate Id and Kind fields in the contents of the artifact - jWithIdAndKind, err := populateIdAndKind(j, content.Kind, content.Metadata.Name) - if err != nil { - return err - } var mimeType string var bytes []byte // Unmarshal the JSON serialization into the message struct. var m proto.Message m, err = mime.MessageForKind(content.Kind) if err == nil { + // First try with id and kind fields included + jWithIdAndKind, err := populateIdAndKind(j, content.Kind, content.Metadata.Name) + if err != nil { + return err + } err = protojson.Unmarshal(jWithIdAndKind, m) if err != nil { if strings.Contains(err.Error(), "unknown field") { // Try unmarshaling the original YAML (without the additional Id and Kind fields). err = protojson.Unmarshal(j, m) - if err != nil { - return err - } } + if err != nil { + return err + } + } else { + j = jWithIdAndKind } + if storeArchivesAsYaml(ctx) { + var node yaml.Node + err = yaml.Unmarshal(j, &node) + if err != nil { + return err + } mimeType = mime.YamlMimeTypeForKind(content.Kind) - encoding.StyleForYAML(&content.Data) - bytes, err = yaml.Marshal(content.Data) + encoding.StyleForYAML(&node) + bytes, err = yaml.Marshal(&node) if err != nil { return err } From 65b4ca5e35c21bff3d2c353089f061609cc4d9aa Mon Sep 17 00:00:00 2001 From: Scott Ganyo Date: Mon, 26 Jun 2023 15:29:51 -0700 Subject: [PATCH 2/3] fix yaml archives for conformance --- cmd/registry/cmd/apply/apply_test.go | 2 - .../cmd/compute/conformance/conformance.go | 5 ++ cmd/registry/patch/artifact.go | 84 ++++--------------- 3 files changed, 19 insertions(+), 72 deletions(-) diff --git a/cmd/registry/cmd/apply/apply_test.go b/cmd/registry/cmd/apply/apply_test.go index 8a9d36088..823bbd100 100644 --- a/cmd/registry/cmd/apply/apply_test.go +++ b/cmd/registry/cmd/apply/apply_test.go @@ -229,8 +229,6 @@ func TestArtifactStorage(t *testing.T) { } want := &apihub.Lifecycle{ - Id: "lifecycle", // deprecated field - Kind: "Lifecycle", // deprecated field DisplayName: "Lifecycle", Description: "A series of stages that an API typically moves through in its lifetime", Stages: []*apihub.Lifecycle_Stage{ diff --git a/cmd/registry/cmd/compute/conformance/conformance.go b/cmd/registry/cmd/compute/conformance/conformance.go index 0b087c317..8c132b77a 100644 --- a/cmd/registry/cmd/compute/conformance/conformance.go +++ b/cmd/registry/cmd/compute/conformance/conformance.go @@ -82,6 +82,11 @@ func Command() *cobra.Command { log.FromContext(ctx).WithError(err).Debugf("Unmarshal() to StyleGuide failed on artifact: %s", artifact.GetName()) return nil } + name, err := names.ParseArtifact(artifact.GetName()) + if err != nil { + return err + } + guide.Id = name.ArtifactID() guides = append(guides, guide) return nil }); err != nil { diff --git a/cmd/registry/patch/artifact.go b/cmd/registry/patch/artifact.go index 8f137e224..9b4f1626b 100644 --- a/cmd/registry/patch/artifact.go +++ b/cmd/registry/patch/artifact.go @@ -16,8 +16,6 @@ package patch import ( "context" - "encoding/json" - "errors" "fmt" "strings" @@ -115,62 +113,29 @@ func storeArchivesAsYaml(ctx context.Context) bool { } func applyArtifactPatch(ctx context.Context, client connection.RegistryClient, content *encoding.Artifact, parent string, filename string) error { - // Restyle the YAML representation so that yaml.Marshal will marshal it as JSON. - encoding.StyleForJSON(&content.Data) - // Marshal the YAML representation into the JSON serialization. - j, err := yaml.Marshal(content.Data) - if err != nil { - return err - } var mimeType string var bytes []byte - // Unmarshal the JSON serialization into the message struct. - var m proto.Message - m, err = mime.MessageForKind(content.Kind) - if err == nil { - // First try with id and kind fields included - jWithIdAndKind, err := populateIdAndKind(j, content.Kind, content.Metadata.Name) + msg, err := mime.MessageForKind(content.Kind) + if storeArchivesAsYaml(ctx) || err != nil { + mimeType = mime.YamlMimeTypeForKind(content.Kind) + encoding.StyleForYAML(&content.Data) + bytes, err = yaml.Marshal(content.Data) if err != nil { return err } - err = protojson.Unmarshal(jWithIdAndKind, m) + } else { + // Convert YAML to JSON for protojson + encoding.StyleForJSON(&content.Data) + j, err := yaml.Marshal(content.Data) if err != nil { - if strings.Contains(err.Error(), "unknown field") { - // Try unmarshaling the original YAML (without the additional Id and Kind fields). - err = protojson.Unmarshal(j, m) - } - if err != nil { - return err - } - } else { - j = jWithIdAndKind + return err } - - if storeArchivesAsYaml(ctx) { - var node yaml.Node - err = yaml.Unmarshal(j, &node) - if err != nil { - return err - } - mimeType = mime.YamlMimeTypeForKind(content.Kind) - encoding.StyleForYAML(&node) - bytes, err = yaml.Marshal(&node) - if err != nil { - return err - } - } else { - mimeType = mime.MimeTypeForKind(content.Kind) - // Marshal the message struct to bytes. - bytes, err = proto.Marshal(m) - if err != nil { - return err - } + err = protojson.Unmarshal(j, msg) + if err != nil { + return err } - } else { - // If there was no struct defined for the type, marshal it struct as YAML mimeType = mime.MimeTypeForKind(content.Kind) - encoding.StyleForYAML(&content.Data) - bytes, err = yaml.Marshal(content.Data) + bytes, err = proto.Marshal(msg) if err != nil { return err } @@ -204,27 +169,6 @@ func applyArtifactPatch(ctx context.Context, client connection.RegistryClient, c return nil } -// populateIdAndKind inserts the "id" and "kind" fields in the supplied json bytes. -func populateIdAndKind(bytes []byte, kind, id string) ([]byte, error) { - var jsonData map[string]interface{} - err := json.Unmarshal(bytes, &jsonData) - if err != nil { - return nil, err - } - if jsonData == nil { - return nil, errors.New("missing data") - } - jsonData["id"] = id - jsonData["kind"] = kind - - rBytes, err := json.Marshal(jsonData) - if err != nil { - return nil, err - } - - return rBytes, nil -} - func UnmarshalContents(contents []byte, mimeType string, message proto.Message) error { if !mime.IsYamlKind(mimeType) { return proto.Unmarshal(contents, message) From 6de7d2afd28acb86602effdcf9173a54467b9961 Mon Sep 17 00:00:00 2001 From: Scott Ganyo Date: Mon, 26 Jun 2023 15:59:54 -0700 Subject: [PATCH 3/3] remove obsolete test --- cmd/registry/patch/patch_test.go | 3 --- .../invalid-artifacts/struct-invalid-structure.yaml | 7 ------- 2 files changed, 10 deletions(-) delete mode 100644 cmd/registry/patch/testdata/invalid-artifacts/struct-invalid-structure.yaml diff --git a/cmd/registry/patch/patch_test.go b/cmd/registry/patch/patch_test.go index 88e2213c4..321ab08d5 100644 --- a/cmd/registry/patch/patch_test.go +++ b/cmd/registry/patch/patch_test.go @@ -1185,9 +1185,6 @@ func TestInvalidArtifactPatches(t *testing.T) { { artifactID: "references-no-data", }, - { - artifactID: "struct-invalid-structure", - }, { artifactID: "struct-no-metadata", }, diff --git a/cmd/registry/patch/testdata/invalid-artifacts/struct-invalid-structure.yaml b/cmd/registry/patch/testdata/invalid-artifacts/struct-invalid-structure.yaml deleted file mode 100644 index d59d1c1ea..000000000 --- a/cmd/registry/patch/testdata/invalid-artifacts/struct-invalid-structure.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: apigeeregistry/v1 -kind: Struct -metadata: - name: struct -data: -value: 42 -children: