diff --git a/README.md b/README.md index 436dc56..abde601 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,13 @@ Generate a sample YAML file from a CRD definition. +## CRD Testing using CTY + +For more information about how to use `cty` for helm-like unit testing your CRD schemas, +please follow the [How to test CRDs with CTY Readme](./crd-testing-README.md). + +![crd-unittest-sample-output](./imgs/crd-unittest-outcome.png) + ## Getting started - Prerequisites: Go installed on your machine. (Check out this link for details: https://go.dev/doc/install) - Clone the repository diff --git a/crd-testing-README.md b/crd-testing-README.md new file mode 100644 index 0000000..c28084a --- /dev/null +++ b/crd-testing-README.md @@ -0,0 +1,141 @@ +# CRD Testing with CTY + +From version `v0.8.0` cty supports the command `test`. + +`test` supports testing CRD schemas against snapshots of generated YAML files that satisfy the schema +or small snippets of yaml strings. + +Adding this test to your CRDs makes sure that any modification on the CRD will not break a generated snapshot of the +CRD. This is from version to version, meaning the tests will make sure that API version is respected. + +## Example + +Let's look at an example. + +Consider the following test suite definition that loosely follows the syntax of helm unittest definitions: + +```yaml +suite: test crd bootstrap +template: crd-bootstrap/crds/bootstrap_crd.yaml # should point to a CRD that is regularly updated like in a helm chart. +tests: + - it: matches bootstrap crds correctly + asserts: + - matchSnapshot: + # this will generate one snapshot / CRD version and match all of them to the right version of the CRD + path: sample-tests/__snapshots__ + - matchSnapshot: + path: sample-tests/__snapshots__ + # generates a yaml file + minimal: true + - it: matches some custom stuff + asserts: + - matchString: + apiVersion: v1alpha1 # this will match this exact version only from the list of versions in the CRD + kind: Bootstrap + spec: + source: + url: + url: https://github.com/Skarlso/test +``` + +Put this into a file called `bootstrap_test.yaml`. + +**IMPORTANT**: `test` will only consider yaml files that end with `_test.yaml`. + +One test is per CRD file. A single CRD file, however, can contain multiple apiVersions. Therefore, it's important to +only target a specific version with a snapshot, otherwise, we might be testing something that is broken intentionally. + +Now, we can run test like this: + +``` +./bin/cty test sample-tests +``` + +The locations in the suite are relative to the execution location. + +Running test like this, will match the following snapshots with the given CRD assuming we have two version v1alpha1 and +v1beta1: +- bootstrap_crd-v1alpha1.yaml +- bootstrap_crd-v1alpha1.min.yaml +- bootstrap_crd-v1beta1.yaml +- bootstrap_crd-v1beta1.min.yaml + +**_Note_**: At this release version the minimal version needs to be adjusted because it will generate an empty object without the closing `{}`. + +If everything is okay, it will generate an output like this: + +``` +./bin/cty test sample-tests ++--------+----------------------------------+---------------+-------+--------------------------------------+ +| STATUS | IT | MATCHER | ERROR | TEMPLATE | ++--------+----------------------------------+---------------+-------+--------------------------------------+ +| PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | +| PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | +| PASS | matches some custom stuff | matchString | | sample-tests/crds/bootstrap_crd.yaml | ++--------+----------------------------------+---------------+-------+--------------------------------------+ + +Tests total: 3, failed: 0, passed: 3 +``` + +If there _was_ an error, it should look something like this: + +``` +./bin/cty test sample-tests ++--------+----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------+ +| STATUS | IT | MATCHER | ERROR | TEMPLATE | ++--------+----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------+ +| PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | +| PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | +| FAIL | matches some custom stuff | matchString | matcher returned failure: failed to validate kind Bootstrap and version v1alpha1 | sample-tests/crds/bootstrap_crd.yaml | +| | | | : spec.source.url in body must be of type object: "null" | | ++--------+----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------+ + +Tests total: 3, failed: 1, passed: 2 +``` + +In the above failing example, we forgot to define the URL field for source. Similarly, if the regex changes for a field +we should error and be alerted that it's a breaking change for any existing users. + +``` +./bin/cty test sample-tests ++--------+-----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------+ +| STATUS | IT | MATCHER | ERROR | TEMPLATE | ++--------+-----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------+ +| PASS | matches AWSCluster crds correctly | matchSnapshot | | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | +| PASS | matches AWSCluster crds correctly | matchSnapshot | | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | +| PASS | matches AWSCluster crds correctly | matchString | | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | +| FAIL | matches AWSCluster crds correctly | matchString | matcher returned failure: failed to validate kind AWSCluster and version v1beta2 | sample-tests/crds/infrastructure.cluster.x-k8s.io_awsclusters.yaml | +| | | | : spec.s3Bucket.name in body should match '^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$' | | +| PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | +| PASS | matches bootstrap crds correctly | matchSnapshot | | sample-tests/crds/bootstrap_crd.yaml | +| PASS | matches some custom stuff | matchString | | sample-tests/crds/bootstrap_crd.yaml | ++--------+-----------------------------------+---------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------+ + +Tests total: 7, failed: 1, passed: 6 +``` + +## Updating Snapshots + +In order to generate snapshots for CRDs, simply add `--update` to the command: + +``` +./bin/cty test sample-tests --update +``` + +It should generate all snapshots and overwrite existing snapshots under the specified folder of the snapshot matcher. +Meaning consider the following yaml snippet from the above test: + +```yaml + asserts: + - matchSnapshot: + # this will generate one snapshot / CRD version and match all of them to the right version of the CRD + path: sample-tests/__snapshots__ +``` + +Provided this `path` the generated snapshots would end up under `sample-tests/__snapshots__` folder with a generated +name that will match the `template` field in the suite. If `template` field is changed, regenerate the tests and +delete any outdated snapshots. + +## Examples + +For further examples, please see under [sample-tests](./sample-tests). diff --git a/docs/release_notes/v0.8.0.md b/docs/release_notes/v0.8.0.md new file mode 100644 index 0000000..1c4bd8d --- /dev/null +++ b/docs/release_notes/v0.8.0.md @@ -0,0 +1,27 @@ +# v0.8.0 + +## MAJOR UPDATE + +### Changes to how values are generated + +This update contains a few modifications to the way we generate samples. These modifications are the following: + +- if enum values are defined for a property, choose the first one from the list whatever that is +- if there is a minimum defined for integer types, the minimum value is used +- comment is added to list items of what type they are and how much the minimum value for them is +```yaml +volumeIDs: [] # minItems 0 of type string +``` +- unless `no-random` is defined, now given a `Pattern` that contains a valid regex a valid value is generated that satisfies the regex + and the regex's value is commented after the value +```yaml +name: xwjhylgy2ruc # ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ +``` + +The random generation can be skipped by providing the following flag to `cty`: `--no-random`. + +### New `test` command + +A new command has been added that lets users unit test schema validation for generated YAML files to CRDs. + +To read more about it, check out the readme: `crd-testing-README.md`. diff --git a/imgs/crd-unittest-outcome.png b/imgs/crd-unittest-outcome.png new file mode 100644 index 0000000..3198e79 Binary files /dev/null and b/imgs/crd-unittest-outcome.png differ diff --git a/pkg/generate.go b/pkg/generate.go index 2f31523..a38dda5 100644 --- a/pkg/generate.go +++ b/pkg/generate.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "io" - "math/rand/v2" "regexp" "slices" "sort" @@ -195,9 +194,7 @@ func outputValueType(v v1beta1.JSONSchemaProps, skipRandom bool) string { } if v.Enum != nil { - i := rand.IntN(len(v.Enum)) //nolint:gosec // enough for our purposes - - return string(v.Enum[i].Raw) + return string(v.Enum[0].Raw) } st := "string" diff --git a/sample-tests/__snapshots__/bootstrap_crd-v1alpha1.yaml b/sample-tests/__snapshots__/bootstrap_crd-v1alpha1.yaml index da8dd24..56e48f8 100644 --- a/sample-tests/__snapshots__/bootstrap_crd-v1alpha1.yaml +++ b/sample-tests/__snapshots__/bootstrap_crd-v1alpha1.yaml @@ -48,9 +48,9 @@ status: - lastTransitionTime: string message: string observedGeneration: 0 - reason: h_AhqJ # ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: "False" - type: 51lg51/V5kVn # ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + reason: i # ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: "True" + type: d # ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ lastAppliedCRDNames: {} lastAppliedRevision: string lastAttemptedRevision: string diff --git a/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta1.yaml b/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta1.yaml index d013f58..67e4026 100644 --- a/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta1.yaml +++ b/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta1.yaml @@ -16,11 +16,11 @@ spec: additionalSecurityGroups: [] # minItems 0 of type string crossZoneLoadBalancing: true healthCheckProtocol: string - name: RxVC # ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ + name: JiHS6tdzS9M # ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ scheme: "internet-facing" subnets: [] # minItems 0 of type string identityRef: - kind: "AWSClusterRoleIdentity" + kind: "AWSClusterControllerIdentity" name: string imageLookupBaseOS: string imageLookupFormat: string @@ -57,7 +57,7 @@ spec: region: string s3Bucket: controlPlaneIAMInstanceProfile: string - name: o.2eu8tg7pq # ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + name: xwjhylgy2ruc # ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ nodesIAMInstanceProfiles: [] # minItems 0 of type string sshKeyName: string status: diff --git a/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta2.yaml b/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta2.yaml index 6c4556e..109943a 100644 --- a/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta2.yaml +++ b/sample-tests/__snapshots__/infrastructure.cluster.x-k8s.io_awsclusters-v1beta2.yaml @@ -16,7 +16,7 @@ spec: additionalSecurityGroups: [] # minItems 0 of type string crossZoneLoadBalancing: true healthCheckProtocol: string - name: GN # ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ + name: QNodCsCWz # ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ scheme: "internet-facing" subnets: [] # minItems 0 of type string identityRef: @@ -57,7 +57,7 @@ spec: region: string s3Bucket: controlPlaneIAMInstanceProfile: string - name: amqq83ljf-wh # ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + name: 3lntyexheg3 # ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ nodesIAMInstanceProfiles: [] # minItems 0 of type string sshKeyName: string status: diff --git a/wasm/app.go b/wasm/app.go index 3e41cfe..4bee528 100644 --- a/wasm/app.go +++ b/wasm/app.go @@ -52,6 +52,7 @@ type Property struct { Default string Required bool Properties []*Property + Enums []string } func (h *crdView) buildError(err error) app.UI { @@ -211,6 +212,9 @@ func render(d app.UI, p []*Property, accordionID string) app.UI { if prop.Required { headerElements = append(headerElements, app.Div().Class("text-bg-primary").Class("col").Text("required")) } + if prop.Enums != nil { + headerElements = append(headerElements, app.Div().Class("text-bg-primary").Class("col").Text(strings.Join(prop.Enums, ","))) + } if prop.Format != "" { headerElements = append(headerElements, app.Div().Class("col").Text(prop.Format)) } @@ -315,6 +319,13 @@ func parseCRD(properties map[string]v1beta1.JSONSchemaProps, version string, req continue } + var enums []string + if v.Enum != nil { + for _, e := range v.Enum { + enums = append(enums, string(e.Raw)) + } + } + p := &Property{ Name: k, Type: v.Type, @@ -324,6 +335,7 @@ func parseCRD(properties map[string]v1beta1.JSONSchemaProps, version string, req Nullable: v.Nullable, Version: version, Required: required, + Enums: enums, } if v.Default != nil { p.Default = string(v.Default.Raw) diff --git a/wasm/index.html b/wasm/index.html index 41438a9..d387b44 100644 --- a/wasm/index.html +++ b/wasm/index.html @@ -2,7 +2,7 @@ - + @@ -10,23 +10,23 @@ - + Preview CRDs - + - - + + - + - + - + @@ -63,7 +63,7 @@
diff --git a/wasm/share.html b/wasm/share.html index e564c88..e92f65d 100644 --- a/wasm/share.html +++ b/wasm/share.html @@ -2,7 +2,7 @@ - + @@ -12,23 +12,23 @@ Preview CRDs - - - - - - - + + + + + + + - - + + - + - + @@ -66,8 +66,8 @@

Oops!

url parameter has to be define in the following format: /share?url=https://example.com/crd.yaml -