From 1b95cf6ffa26197e174c348a493cdef9c1feb689 Mon Sep 17 00:00:00 2001 From: Colin Dean Date: Mon, 28 Aug 2023 16:20:11 -0400 Subject: [PATCH] Increase Starlark execution limit, abstract limit resolver 5,000 was too few to enable the example added to the testdata to work, so was 6,000. I chose 7,500 arbitrarily after a test at 10,000 and both worked. In the long term, this should probably be configurable so as not to require recompilation. For now, this kicks the can down the road while allowing this build matrix use case to exist. --- compiler/template/starlark/render.go | 17 +- compiler/template/starlark/render_test.go | 6 +- .../starlark/testdata/build/large/build.star | 202 ++++++++++++++++++ 3 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 compiler/template/starlark/testdata/build/large/build.star diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index 0be1063f5..96b2f1b3d 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -38,7 +38,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM // arbitrarily limiting the steps of the thread to 5000 to help prevent infinite loops // may need to further investigate spawning a separate POSIX process if user input is problematic // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details - thread.SetMaxExecutionSteps(5000) + thread.SetMaxExecutionSteps(GetStarlarkExecutionStepLimit()) predeclared := starlark.StringDict{"struct": starlark.NewBuiltin("struct", starlarkstruct.Make)} @@ -136,6 +136,15 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil } +// GetStarlarkExecutionStepLimit may eventually look up config or calculate it +func GetStarlarkExecutionStepLimit() uint64 { + // arbitrarily limiting the steps of the thread to help prevent infinite loops + // may need to further investigate spawning a separate POSIX process if user input is problematic + // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details + // This value was previously 5000 and that inhibited a four-dimensional build matrix from working. + return 7500 +} + // RenderBuild renders the templated build. // //nolint:lll // ignore function length due to input args @@ -143,10 +152,8 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st config := new(types.Build) thread := &starlark.Thread{Name: "templated-base"} - // arbitrarily limiting the steps of the thread to 5000 to help prevent infinite loops - // may need to further investigate spawning a separate POSIX process if user input is problematic - // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details - thread.SetMaxExecutionSteps(5000) + + thread.SetMaxExecutionSteps(GetStarlarkExecutionStepLimit()) predeclared := starlark.StringDict{"struct": starlark.NewBuiltin("struct", starlarkstruct.Make)} diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index 8cc089e47..c602bac38 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -92,6 +92,8 @@ func TestStarlark_Render(t *testing.T) { } func TestNative_RenderBuild(t *testing.T) { + noWantFile := "none" + type args struct { velaFile string } @@ -106,6 +108,7 @@ func TestNative_RenderBuild(t *testing.T) { {"stages", args{velaFile: "testdata/build/basic_stages/build.star"}, "testdata/build/basic_stages/want.yml", false}, {"conditional match", args{velaFile: "testdata/build/conditional/build.star"}, "testdata/build/conditional/want.yml", false}, {"steps, with structs", args{velaFile: "testdata/build/with_struct/build.star"}, "testdata/build/with_struct/want.yml", false}, + {"large build to stress execution steps", args{velaFile: "testdata/build/large/build.star"}, noWantFile, false}, } for _, tt := range tests { @@ -118,13 +121,14 @@ func TestNative_RenderBuild(t *testing.T) { got, err := RenderBuild("build", string(sFile), map[string]string{ "VELA_REPO_FULL_NAME": "octocat/hello-world", "VELA_BUILD_BRANCH": "master", + "VELA_REPO_ORG": "octocat", }, map[string]interface{}{}) if (err != nil) != tt.wantErr { t.Errorf("RenderBuild() error = %v, wantErr %v", err, tt.wantErr) return } - if tt.wantErr != true { + if tt.wantErr != true && tt.wantFile != noWantFile { wFile, err := os.ReadFile(tt.wantFile) if err != nil { t.Error(err) diff --git a/compiler/template/starlark/testdata/build/large/build.star b/compiler/template/starlark/testdata/build/large/build.star new file mode 100644 index 000000000..5ffbc482a --- /dev/null +++ b/compiler/template/starlark/testdata/build/large/build.star @@ -0,0 +1,202 @@ +###### +## Setup the build matrix with the base versions a human will maintain. +###### + +DISTRO_WITH_VERSIONS = { + # n.b. these reduce to DockerHub tags + # https://hub.docker.com/_/python/tags?name=alpine + # https://endoflife.date/alpine + 'alpine': [ + '3.17', # EOL 22 Nov 2024 + '3.18' # EOL 09 May 2025 + ], + # https://hub.docker.com/_/python/tags?name=slim + # https://endoflife.date/debian + 'debian': [ + 'slim-bullseye', # EOL 30 Jun 2026 + 'slim-bookworm' # EOL 10 Jun 2028 + ] +} +PYTHON_VERSIONS = [ + '3.8', + '3.9', + '3.10', + '3.11' +] +POETRY_VERSIONS = [ + '1.6.1' +] + +KANIKO_IMAGE = 'target/vela-kaniko:latest' +HADOLINT_IMAGE = 'hadolint/hadolint:v2.12.0-alpine' + + +## The base Docker container build step's config for push builds +def base(): + return { + 'image': KANIKO_IMAGE, + 'ruleset': { + 'event': 'push', + 'branch': 'main' + }, + 'pull': 'not_present', + 'secrets': [ + { + 'source': 'artifactory_password', + 'target': 'docker_password' + } + ] + } + + +## The base Docker container plugin params for push builds +## +## These are parameters passed to Kaniko. +def base_params(): + return { + 'username': 'ibuildallthings', + 'registry': 'docker.example.com', + 'repo': 'docker.example.com/app/multibuild' + } + + +## The step config for pull request builds +def pull_request(): + pr = base() + pr['ruleset']['event'] = 'pull_request' + pr['ruleset'].pop('branch') + return pr + + +## The Kaniko params for pull request builds +def pull_request_params(): + prp = base_params() + prp['dry_run'] = True + return prp + + +## Define a linting stage that uses Hadolint inside of a Make task +## +## This keeps our Dockerfiles tidy and compliant with conventions +def stage_linting(): + return { + 'linting': { + 'steps': [{ + 'name': 'check-docker', + 'image': HADOLINT_IMAGE, + 'pull': 'not_present', + 'commands': [ + 'time apk add --no-cache make', + 'time make check-docker' + ] + }] + } + } + + +## Build stages comprised of a step for push and pull_request builds +def stage_build_tuple(distro, distro_version, python_version, poetry_version): + pr = build_template("build", distro, distro_version, python_version, poetry_version, pull_request(), pull_request_params()) + base_step = build_template("publish", distro, distro_version, python_version, poetry_version, base(), base_params()) + combined = base_step | pr + return combined + + +## Build a single stage for a build tuple, with its base step config and plugin parameters +def build_template(step_name, distro, distro_version, python_version, poetry_version, step_def_base, step_def_params): + return { + ('python_%s_%s_%s %s' % (python_version, distro, distro_version, step_def_base['ruleset']['event'])): { + 'steps': [step_def_base | { + 'name': ('%s python-%s %s %s' % (step_name, python_version, distro, distro_version)), + 'parameters': step_def_params | { + 'dockerfile': ('python-%s.Dockerfile' % distro), + 'build_args': [ + 'PYTHON_VERSION=%s' % python_version, + '%s_VERSION=%s' % (distro.upper(), distro_version), + 'POETRY_VERSION=%s' % poetry_version + ], + 'tags': [ + '%s-%s-%s-%s' % (python_version, distro, distro_version, poetry_version), + '%s-%s-%s' % (python_version, distro, distro_version), + '%s-%s' % (python_version, distro) + ] + } + }] + } + } + + +## Define a stage that uses the Slack template +def stage_slack_notify(needs): + return { + 'slack': { + 'needs': needs, + 'steps': [{ + 'name': 'slack', + 'template': { + 'name': 'slack' + } + }] + } + } + + +## Builds the build matrix in the form of list of tuples from the constants defined at the top of the file +def build_matrix(): + BUILD_MATRIX = [] + for poetry_version in POETRY_VERSIONS: + for python_version in PYTHON_VERSIONS: + for distro in DISTRO_WITH_VERSIONS: + for distro_version in DISTRO_WITH_VERSIONS[distro]: + BUILD_MATRIX.append((distro, + distro_version, + python_version, + poetry_version)) + return BUILD_MATRIX + + +## Construct a secret +def secret(name, key, secret_type, engine='native'): + return {'name': name, 'key': key, 'engine': engine, 'type': secret_type} + + +## Construct a template +def template(name, source, version=None, template_type='github'): + real_source = '%s@%s' % (source, version) if version else source + return { + 'name': name, + 'source': real_source, + 'type': template_type + } + +## The main method, the real deal. +## +## Vela actually calls this function, its return is what Vela uses. +def main(ctx): + # Retrieve the org dynamically since we're using some org secrets + vela_repo_org = ctx['vela']['repo']['org'] if 'vela' in ctx else "UNKNOWN-ORG" + + # Build the stages from the build matrix + build_stages = {} + for (distro, distro_version, python_version, poetry_version) in build_matrix(): + build_stages = build_stages | (stage_build_tuple(distro, distro_version, python_version, poetry_version)) + + # assemble the stage list with the bookends of linting and notifications in place + stages = stage_linting() | build_stages | stage_slack_notify(build_stages.keys()) + + # Build the final output + final = { + 'version': '1', + 'templates': [ + template(name='slack', + source='git.example.com/vela/vela-templates/slack/slack.yml') + ], + 'stages': stages, + 'secrets': [ + secret('artifactory_password','platform/vela-secrets/artifactory_password_for_ibuildallthings', 'shared'), + secret('slack_webhook', vela_repo_org + '/slack_webhook', 'org') + ] + } + + return final +