From 6af385b26200e86d309ec1cdfc0690b62f0adb83 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Wed, 1 Jun 2022 14:09:47 -0700 Subject: [PATCH 1/9] test make test Signed-off-by: Sean Lin --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 65ab125cf..9e8323b8e 100755 --- a/Makefile +++ b/Makefile @@ -10,3 +10,4 @@ update_boilerplate: generate: download_tooling @go generate ./... + From c672cd0c72ad79bb6fef72dc24460bafa6d981c2 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Wed, 1 Jun 2022 15:41:23 -0700 Subject: [PATCH 2/9] test pin golangci-lint version Signed-off-by: Sean Lin --- boilerplate/flyte/golang_test_targets/download_tooling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh index 003220ea7..a1eeaccae 100755 --- a/boilerplate/flyte/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -18,7 +18,7 @@ set -e tools=( "github.com/EngHabu/mockery/cmd/mockery" "github.com/flyteorg/flytestdlib/cli/pflags@latest" - "github.com/golangci/golangci-lint/cmd/golangci-lint@latest" + "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2" "github.com/alvaroloes/enumer" "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) From 73d65f3487551082e155d84087c343c0ce595254 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Wed, 1 Jun 2022 18:24:21 -0700 Subject: [PATCH 3/9] Test upgrade to go 1.18 Signed-off-by: Sean Lin --- .../golang_test_targets/download_tooling.sh | 2 +- go.mod | 108 ++++++++++++++++-- go.sum | 1 - 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh index a1eeaccae..003220ea7 100755 --- a/boilerplate/flyte/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -18,7 +18,7 @@ set -e tools=( "github.com/EngHabu/mockery/cmd/mockery" "github.com/flyteorg/flytestdlib/cli/pflags@latest" - "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2" + "github.com/golangci/golangci-lint/cmd/golangci-lint@latest" "github.com/alvaroloes/enumer" "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) diff --git a/go.mod b/go.mod index 400ced060..6b0e51cb4 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,10 @@ module github.com/flyteorg/flyteplugins -go 1.16 +go 1.18 require ( - cloud.google.com/go v0.78.0 // indirect github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20200723154620-6f35a1152625 github.com/Masterminds/semver v1.5.0 - github.com/adammck/venv v0.0.0-20200610172036-e77789703e7c // indirect github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d github.com/aws/aws-sdk-go v1.37.3 github.com/aws/aws-sdk-go-v2 v1.2.0 @@ -15,11 +13,8 @@ require ( github.com/coocood/freecache v1.1.1 github.com/flyteorg/flyteidl v1.0.0 github.com/flyteorg/flytestdlib v1.0.0 - github.com/go-logr/zapr v0.4.0 // indirect github.com/go-test/deep v1.0.7 github.com/golang/protobuf v1.4.3 - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.2.0 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/imdario/mergo v0.3.11 github.com/kubeflow/common v0.4.0 @@ -32,12 +27,8 @@ require ( github.com/prometheus/client_golang v1.10.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.16.0 // indirect golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93 - golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/api v0.40.0 google.golang.org/grpc v1.35.0 google.golang.org/protobuf v1.25.0 @@ -49,4 +40,101 @@ require ( sigs.k8s.io/controller-runtime v0.8.2 ) +require ( + cloud.google.com/go v0.78.0 // indirect + cloud.google.com/go/storage v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.17 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.0 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/adammck/venv v0.0.0-20200610172036-e77789703e7c // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.0.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.0.0 // indirect + github.com/aws/smithy-go v1.1.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful v2.9.6+incompatible // indirect + github.com/evanphx/json-patch v4.9.0+incompatible // indirect + github.com/fatih/color v1.10.0 // indirect + github.com/flyteorg/stow v0.3.3 // indirect + github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-logr/zapr v0.4.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/spec v0.20.3 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gofrs/uuid v4.2.0+incompatible // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/googleapis/gnostic v0.5.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.10 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/ncw/swift v1.0.53 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.18.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/sirupsen/logrus v1.7.0 // indirect + github.com/spf13/afero v1.5.1 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cobra v1.1.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/viper v1.7.1 // indirect + github.com/stretchr/objx v0.3.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + go.opencensus.io v0.22.6 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.16.0 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + golang.org/x/tools v0.1.10 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/klog/v2 v2.5.0 // indirect + k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.0.3 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) + replace github.com/aws/amazon-sagemaker-operator-for-k8s => github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20210303003444-0fb33b1fd49d diff --git a/go.sum b/go.sum index d95f05373..35baa7271 100644 --- a/go.sum +++ b/go.sum @@ -1331,7 +1331,6 @@ k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= From a188895e6eab6bc8bdd39b61694fd5f6b6ad138c Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Wed, 1 Jun 2022 21:50:45 -0700 Subject: [PATCH 4/9] Switch to actions/setup-go for golang 1.18 support Signed-off-by: Sean Lin --- .github/workflows/pull_request.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 35bf213f0..d50f6cd7c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -9,12 +9,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Unit Tests - uses: cedrickring/golang-action@1.7.0 - env: - GO111MODULE: "on" + - uses: actions/setup-go@v2 with: - args: make install && make test_unit_codecov + go-version: '1.18' + - name: Unit Tests + run: make install && make test_unit_codecov - name: Push CodeCov uses: codecov/codecov-action@v1 with: @@ -34,6 +33,6 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: '1.18' - name: Go generate and diff run: DELTA_CHECK=true make generate From 6cdc116e446529cc8885ab624bf8c97b9e00db7b Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Wed, 1 Jun 2022 22:05:12 -0700 Subject: [PATCH 5/9] Fix lint Signed-off-by: Sean Lin --- .github/workflows/pull_request.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d50f6cd7c..df80f88a6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,11 +21,7 @@ jobs: flags: unittests fail_ci_if_error: true - name: Lint - uses: cedrickring/golang-action@1.7.0 - env: - GO111MODULE: "on" - with: - args: make install && make lint + run: make install && make lint generate: runs-on: ubuntu-latest From c720e41880e130a2c27ff7901a80d526e6b50a7a Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Thu, 2 Jun 2022 00:51:03 -0700 Subject: [PATCH 6/9] test nolint Signed-off-by: Sean Lin --- go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go | 3 +-- go/tasks/plugins/array/k8s/subtask_exec_context.go | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go b/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go index 781cdbf41..e560c2b43 100755 --- a/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go +++ b/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go @@ -2,7 +2,6 @@ package flytek8s import ( "context" - "fmt" "os" "reflect" "testing" @@ -250,7 +249,7 @@ func TestDecorateEnvVars(t *testing.T) { originalEnvVal := os.Getenv("value") err := os.Setenv("value", "v") if err != nil { - t.Fatal(fmt.Sprintf("failed to set env var 'value'; %v", err)) + t.Fatalf("failed to set env var 'value'; %v", err) } defer os.Setenv("value", originalEnvVal) diff --git a/go/tasks/plugins/array/k8s/subtask_exec_context.go b/go/tasks/plugins/array/k8s/subtask_exec_context.go index 37e2f3460..ba389ca6a 100644 --- a/go/tasks/plugins/array/k8s/subtask_exec_context.go +++ b/go/tasks/plugins/array/k8s/subtask_exec_context.go @@ -60,9 +60,11 @@ func NewSubTaskExecutionContext(ctx context.Context, tCtx pluginsCore.TaskExecut } // construct TaskTemplate + //nolint subtaskTemplate := &core.TaskTemplate{} *subtaskTemplate = *taskTemplate + //nolint if subtaskTemplate != nil { subtaskTemplate.TaskTypeVersion = 2 if subtaskTemplate.GetContainer() != nil { From e88820d56a4d7f6b0aeb9058756f3def47872000 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Thu, 2 Jun 2022 00:56:51 -0700 Subject: [PATCH 7/9] Resolve SA4031: this nil check is always true Signed-off-by: Sean Lin --- go/tasks/plugins/array/k8s/subtask_exec_context.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/go/tasks/plugins/array/k8s/subtask_exec_context.go b/go/tasks/plugins/array/k8s/subtask_exec_context.go index ba389ca6a..0eb0a92e8 100644 --- a/go/tasks/plugins/array/k8s/subtask_exec_context.go +++ b/go/tasks/plugins/array/k8s/subtask_exec_context.go @@ -60,18 +60,14 @@ func NewSubTaskExecutionContext(ctx context.Context, tCtx pluginsCore.TaskExecut } // construct TaskTemplate - //nolint subtaskTemplate := &core.TaskTemplate{} *subtaskTemplate = *taskTemplate - //nolint - if subtaskTemplate != nil { - subtaskTemplate.TaskTypeVersion = 2 - if subtaskTemplate.GetContainer() != nil { - subtaskTemplate.Type = podPlugin.ContainerTaskType - } else if taskTemplate.GetK8SPod() != nil { - subtaskTemplate.Type = podPlugin.SidecarTaskType - } + subtaskTemplate.TaskTypeVersion = 2 + if subtaskTemplate.GetContainer() != nil { + subtaskTemplate.Type = podPlugin.ContainerTaskType + } else if taskTemplate.GetK8SPod() != nil { + subtaskTemplate.Type = podPlugin.SidecarTaskType } arrayInputReader := array.GetInputReader(tCtx, taskTemplate) From 430a3d3bf3314acb1e21ab10c1632edaef337c52 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Thu, 2 Jun 2022 15:28:47 -0700 Subject: [PATCH 8/9] Upgrade to go 1.18 in github workflows Signed-off-by: Sean Lin --- .github/workflows/master.yml | 14 +++----------- .github/workflows/pull_request.yml | 3 ++- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index bf25533eb..c3483d010 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -34,7 +34,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: '1.18' - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: @@ -50,11 +50,7 @@ jobs: with: fetch-depth: "0" - name: Unit Tests - uses: cedrickring/golang-action@1.7.0 - env: - GO111MODULE: "on" - with: - args: make install && make test_unit_codecov + run: make install && make test_unit_codecov - name: Push CodeCov uses: codecov/codecov-action@v1 with: @@ -62,8 +58,4 @@ jobs: flags: unittests fail_ci_if_error: true - name: Lint - uses: cedrickring/golang-action@1.7.0 - env: - GO111MODULE: "on" - with: - args: make install && make lint + run: make install && make lint diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index df80f88a6..c6c000639 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -9,7 +9,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v2 with: go-version: '1.18' - name: Unit Tests From b46a28eb7a5f8d601f67374ea7e4d6168db717e7 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Thu, 2 Jun 2022 17:12:26 -0700 Subject: [PATCH 9/9] minor fix Signed-off-by: Sean Lin --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 9e8323b8e..65ab125cf 100755 --- a/Makefile +++ b/Makefile @@ -10,4 +10,3 @@ update_boilerplate: generate: download_tooling @go generate ./... -