From 415ce00e2454897e10e904e98a403e75ccd2de94 Mon Sep 17 00:00:00 2001 From: Mateusz Szostok Date: Thu, 21 Apr 2022 09:31:12 +0200 Subject: [PATCH 1/3] Narrow-down the download TypeInstance details --- pkg/argo-actions/download_type_instances.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/argo-actions/download_type_instances.go b/pkg/argo-actions/download_type_instances.go index de0e3b5b6..20383e243 100644 --- a/pkg/argo-actions/download_type_instances.go +++ b/pkg/argo-actions/download_type_instances.go @@ -4,11 +4,13 @@ import ( "context" "fmt" - hubclient "capact.io/capact/pkg/hub/client" - "capact.io/capact/pkg/runner" "github.com/pkg/errors" "go.uber.org/zap" "sigs.k8s.io/yaml" + + hubclient "capact.io/capact/pkg/hub/client" + "capact.io/capact/pkg/hub/client/local" + "capact.io/capact/pkg/runner" ) // DownloadAction represents the download TypeInstance action. @@ -52,7 +54,7 @@ func NewDownloadAction(log *zap.Logger, client *hubclient.Client, cfg []Download func (d *Download) Do(ctx context.Context) error { for _, config := range d.cfg { d.log.Info("Downloading TypeInstance", zap.String("ID", config.ID), zap.String("Path", config.Path)) - typeInstance, err := d.client.FindTypeInstance(ctx, config.ID) + typeInstance, err := d.client.FindTypeInstance(ctx, config.ID, local.WithFields(local.TypeInstanceLatestResourceVersionFields)) if err != nil { return err } From 743c46ee1870682f621ad24251f1014e596a2377 Mon Sep 17 00:00:00 2001 From: Mateusz Szostok Date: Thu, 21 Apr 2022 17:23:24 +0200 Subject: [PATCH 2/3] Change goTemplate to string in Terraform runner --- cmd/terraform-runner/example-input/args.yml | 2 +- .../terraform-implementation.yaml.tmpl | 2 +- ...cap.implementation.terraform.aws.test.yaml | 2 +- ...cap.implementation.terraform.gcp.test.yaml | 2 +- ...implementation.terraform.generic.test.yaml | 2 +- .../cap.implementation.terraform.test.yaml | 180 ------------------ pkg/runner/terraform/terraform.go | 11 +- pkg/runner/terraform/types.go | 4 +- 8 files changed, 8 insertions(+), 197 deletions(-) delete mode 100644 internal/cli/manifestgen/testdata/cap.implementation.terraform.test.yaml diff --git a/cmd/terraform-runner/example-input/args.yml b/cmd/terraform-runner/example-input/args.yml index 5e22bd6f2..d31e40772 100644 --- a/cmd/terraform-runner/example-input/args.yml +++ b/cmd/terraform-runner/example-input/args.yml @@ -11,7 +11,7 @@ env: # so setting it directly - GOOGLE_PROJECT=capact output: - goTemplate: + goTemplate: | host: "{{ .instance_ip_addr }}" port: 5432 defaultDBName: postgres diff --git a/internal/cli/manifestgen/templates/terraform-implementation.yaml.tmpl b/internal/cli/manifestgen/templates/terraform-implementation.yaml.tmpl index bc09128a0..c522109a1 100644 --- a/internal/cli/manifestgen/templates/terraform-implementation.yaml.tmpl +++ b/internal/cli/manifestgen/templates/terraform-implementation.yaml.tmpl @@ -137,7 +137,7 @@ spec: - GOOGLE_PROJECT=<@ providercredentials.project_id @> - GOOGLE_APPLICATION_CREDENTIALS=/additional{{else}}[]{{end}} output: - goTemplate: + goTemplate: | {{- range $index, $output := .Outputs }} {{ $output.Name }}: "{{`{{`}} .{{ $output.Name }} {{`}}`}}" {{- end }} diff --git a/internal/cli/manifestgen/testdata/cap.implementation.terraform.aws.test.yaml b/internal/cli/manifestgen/testdata/cap.implementation.terraform.aws.test.yaml index 17c3b82fc..890da906e 100644 --- a/internal/cli/manifestgen/testdata/cap.implementation.terraform.aws.test.yaml +++ b/internal/cli/manifestgen/testdata/cap.implementation.terraform.aws.test.yaml @@ -105,7 +105,7 @@ spec: - AWS_ACCESS_KEY_ID=<@ providercredentials.accessKeyID @> - AWS_SECRET_ACCESS_KEY=<@ providercredentials.secretAccessKey @> output: - goTemplate: + goTemplate: | instance_ids: "{{ .instance_ids }}" random_number: "{{ .random_number }}" variables: |+ diff --git a/internal/cli/manifestgen/testdata/cap.implementation.terraform.gcp.test.yaml b/internal/cli/manifestgen/testdata/cap.implementation.terraform.gcp.test.yaml index ab4ac0b82..be1a3dbf2 100644 --- a/internal/cli/manifestgen/testdata/cap.implementation.terraform.gcp.test.yaml +++ b/internal/cli/manifestgen/testdata/cap.implementation.terraform.gcp.test.yaml @@ -112,7 +112,7 @@ spec: - GOOGLE_PROJECT=<@ providercredentials.project_id @> - GOOGLE_APPLICATION_CREDENTIALS=/additional output: - goTemplate: + goTemplate: | instance_ids: "{{ .instance_ids }}" random_number: "{{ .random_number }}" variables: |+ diff --git a/internal/cli/manifestgen/testdata/cap.implementation.terraform.generic.test.yaml b/internal/cli/manifestgen/testdata/cap.implementation.terraform.generic.test.yaml index 86823fca0..b291cbe66 100644 --- a/internal/cli/manifestgen/testdata/cap.implementation.terraform.generic.test.yaml +++ b/internal/cli/manifestgen/testdata/cap.implementation.terraform.generic.test.yaml @@ -98,7 +98,7 @@ spec: source: "https://example.com/module.tgz" env: [] output: - goTemplate: + goTemplate: | instance_ids: "{{ .instance_ids }}" random_number: "{{ .random_number }}" variables: |+ diff --git a/internal/cli/manifestgen/testdata/cap.implementation.terraform.test.yaml b/internal/cli/manifestgen/testdata/cap.implementation.terraform.test.yaml deleted file mode 100644 index c307974e9..000000000 --- a/internal/cli/manifestgen/testdata/cap.implementation.terraform.test.yaml +++ /dev/null @@ -1,180 +0,0 @@ -ocfVersion: 0.0.1 -revision: 0.1.0 -kind: Implementation -metadata: - prefix: "cap.implementation.terraform" - name: test - displayName: "test Action" - description: "test Action" - documentationURL: https://example.com - supportURL: https://example.com - maintainers: - - email: dev@example.com - name: Example Dev - url: https://example.com - license: - name: "Apache 2.0" - -spec: - appVersion: "1.0.x" # TODO(ContentDeveloper): Set the supported application version here - additionalInput: - parameters: - typeRef: - path: "cap.type.terraform.test-input-parameters" - revision: 0.1.0 - - outputTypeInstanceRelations: - config: - uses: - - terraform-release - - implements: - - path: cap.interface.group.test - revision: 0.2.0 - - requires: - cap.type.aws.auth: - allOf: - - name: credentials - alias: aws-credentials - revision: 0.1.0 - - imports: - - interfaceGroupPath: cap.interface.runner.argo - alias: argo - methods: - - name: run - revision: 0.1.0 - - interfaceGroupPath: cap.interface.templating.jinja2 - alias: jinja2 - methods: - - name: template - revision: 0.1.0 - - interfaceGroupPath: cap.interface.runner.terraform - alias: terraform - methods: - - name: apply - revision: 0.1.0 - - action: - runnerInterface: argo.run - args: - workflow: - entrypoint: deploy - templates: - - name: deploy - inputs: - artifacts: - - name: input-parameters - - name: additional-parameters - optional: true - outputs: - artifacts: [] - steps: - - - name: fill-default-input - capact-action: jinja2.template - arguments: - artifacts: - - name: input-parameters - from: "{{inputs.artifacts.input-parameters}}" - - name: template - raw: - # TODO(ContentDeveloper): Put the input parameters from the Interface here and set default values for it: - data: | - my_property: <@ input.my_property | default("default_value") @> - - name: configuration - raw: - data: | - prefix: input - - - - name: create-module-args - capact-action: jinja2.template - arguments: - artifacts: - - name: input-parameters - from: "{{inputs.artifacts.additional-parameters}}" - - name: configuration - raw: - data: | - prefix: additionalInput - - name: template - raw: - data: | - command: "apply" - module: - name: "test" - source: "https://example.com/module.tgz" - env: - - AWS_ACCESS_KEY_ID=<@ creds.accessKeyID @> - - AWS_SECRET_ACCESS_KEY=<@ creds.secretAccessKey @> - output: - goTemplate: - instance_ids: "{{ .instance_ids }}" - random_number: "{{ .random_number }}" - - variables: |+ - <% if additionalInput.count -%> - count = "<@ additionalInput.count @>" - <%- endif %> - - <% if additionalInput.name -%> - name = "<@ additionalInput.name @>" - <%- endif %> - - - - - name: fill-parameters - capact-action: jinja2.template - arguments: - artifacts: - - name: template - from: "{{steps.create-module-args.outputs.artifacts.render}}" - - name: input-parameters - from: "{{steps.fill-default-input.outputs.artifacts.render}}" - - name: configuration - raw: - data: | - prefix: input - - - - name: fill-creds - capact-action: jinja2.template - arguments: - artifacts: - - name: template - from: "{{steps.fill-parameters.outputs.artifacts.render}}" - - name: input-parameters - from: "{{workflow.outputs.artifacts.aws-credentials}}" - - name: configuration - raw: - data: | - prefix: creds - - - - name: terraform-apply - capact-action: terraform.apply - capact-outputTypeInstances: - - name: terraform-release - from: terraform-release - arguments: - artifacts: - - name: input-parameters - from: "{{steps.fill-creds.outputs.artifacts.render}}" - - name: runner-context - from: "{{workflow.outputs.artifacts.runner-context}}" - - - - name: render-config - capact-outputTypeInstances: - - name: config - from: render - capact-action: jinja2.template - arguments: - artifacts: - - name: input-parameters - from: "{{steps.terraform-apply.outputs.artifacts.additional}}" - - name: configuration - raw: - data: "" - - name: template - raw: - # TODO(ContentDeveloper): Fill the properties of the output TypeInstance here - data: | - property: value - \ No newline at end of file diff --git a/pkg/runner/terraform/terraform.go b/pkg/runner/terraform/terraform.go index a4eae827f..afe38c6b2 100644 --- a/pkg/runner/terraform/terraform.go +++ b/pkg/runner/terraform/terraform.go @@ -153,21 +153,14 @@ func (t *terraform) variables() ([]byte, error) { } func (t *terraform) renderOutput() ([]byte, error) { - if t.args.Output.GoTemplate == nil { + if t.args.Output.GoTemplate == "" { return []byte{}, nil } if len(t.runOutput) == 0 { return []byte{}, nil } - // yaml.Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object - // but the GoTemplate is defined via YAML, so we need to revert that change - artifactTemplate, err := yaml.JSONToYAML(t.args.Output.GoTemplate) - if err != nil { - return nil, errors.Wrap(err, "while converting GoTemplate property from JSON to YAML") - } - - tmpl, err := template.New("output").Parse(string(artifactTemplate)) + tmpl, err := template.New("output").Parse(t.args.Output.GoTemplate) if err != nil { return nil, errors.Wrap(err, "failed to load template") } diff --git a/pkg/runner/terraform/types.go b/pkg/runner/terraform/types.go index 086860449..cfe321a84 100644 --- a/pkg/runner/terraform/types.go +++ b/pkg/runner/terraform/types.go @@ -1,7 +1,5 @@ package terraform -import "encoding/json" - // CommandType represents the operation type to be performed by the runner. type CommandType string @@ -33,7 +31,7 @@ type Module struct { // AdditionalOutput stores input arguments for generating the additional output. type AdditionalOutput struct { - GoTemplate json.RawMessage `yaml:"goTemplate"` + GoTemplate string `yaml:"goTemplate"` } // Config holds Runner related configuration. From 2bd5595ba11195e943272a77ee4691929bc06cb2 Mon Sep 17 00:00:00 2001 From: Mateusz Szostok Date: Wed, 27 Apr 2022 11:48:22 +0200 Subject: [PATCH 3/3] Add required delay for value fetcher --- cmd/ti-value-fetcher/main.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/ti-value-fetcher/main.go b/cmd/ti-value-fetcher/main.go index 951b98e4a..f4f3737fb 100644 --- a/cmd/ti-value-fetcher/main.go +++ b/cmd/ti-value-fetcher/main.go @@ -3,13 +3,15 @@ package main import ( "fmt" "log" + "time" "google.golang.org/grpc" - "capact.io/capact/internal/logger" - tivaluefetcher "capact.io/capact/internal/ti-value-fetcher" "github.com/vrischmann/envconfig" "sigs.k8s.io/controller-runtime/pkg/manager/signals" + + "capact.io/capact/internal/logger" + tivaluefetcher "capact.io/capact/internal/ti-value-fetcher" ) // Config holds TypeInstance Value resolver configuration. @@ -26,6 +28,8 @@ type Config struct { const appName = "ti-value-fetcher" func main() { + start := time.Now() + var cfg Config err := envconfig.InitWithPrefix(&cfg, "APP") exitOnError(err, "while loading configuration") @@ -48,6 +52,13 @@ func main() { err = tiValueFetcher.SaveToFile(cfg.OutputFilePath, res) exitOnError(err, fmt.Sprintf("while saving output to file %q", cfg.OutputFilePath)) + + // Argo doesn't like when a Pod exits too fast + // and this may happen when value already provided and no further logic is executed + minTime := start.Add(time.Second) + if time.Now().Before(minTime) { + time.Sleep(time.Second) + } } func exitOnError(err error, context string) {