From 09c8662702bbdcb23830c2d074ac7ce9662fc38a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnau=20Alc=C3=A1zar=20Lleopart?= Date: Tue, 22 Oct 2024 14:15:49 +0200 Subject: [PATCH 01/54] GHA to submit dependency graph to Github (#715) * GHA to submit dependency graph to Github * Generate graph on every push to master * change tower to wave Signed-off-by: munishchouhan * changed for testing Signed-off-by: munishchouhan * Revert "changed for testing" This reverts commit 6fee4f944a9f4bacf56fd5979edd93d24cbdcc97. --------- Signed-off-by: munishchouhan Co-authored-by: munishchouhan --- .../security-submit-dependecy-graph.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/security-submit-dependecy-graph.yml diff --git a/.github/workflows/security-submit-dependecy-graph.yml b/.github/workflows/security-submit-dependecy-graph.yml new file mode 100644 index 000000000..e2c3a7aa7 --- /dev/null +++ b/.github/workflows/security-submit-dependecy-graph.yml @@ -0,0 +1,24 @@ +name: Generate and submit dependency graph for wave +on: + push: + branches: ['master'] + +permissions: + contents: write + +jobs: + dependency-submission: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Generate and submit dependency graph for wave + uses: gradle/actions/dependency-submission@v4 + with: + dependency-resolution-task: "dependencies" + additional-arguments: "--configuration runtimeClasspath" + dependency-graph: generate-and-submit From 4473fe8c174037b589a36e0e87d73a9b0060ee59 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 23 Oct 2024 09:35:38 +0200 Subject: [PATCH 02/54] Fix prevent scan on cached failed builds] Signed-off-by: Paolo Di Tommaso --- .../scan/ContainerScanServiceImpl.groovy | 4 ++-- .../scan/ContainerScanServiceImplTest.groovy | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 62b089c3b..b328d271d 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -136,12 +136,12 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler> BUILD_ID request.buildNew >> BUILD_NEW request.dryRun >> DRY_RUN + request.succeeded >> SUCCEEDED and: def scan = Mock(ScanRequest) @@ -412,15 +413,16 @@ class ContainerScanServiceImplTest extends Specification { RUN_TIMES * scanService.scan(scan) >> null where: - SCAN_ID | BUILD_ID | BUILD_NEW | DRY_RUN | EXISTS_SCAN | RUN_TIMES - null | null | null | null | false | 0 - 'sc-123'| null | null | null | false | 0 + SCAN_ID | BUILD_ID | BUILD_NEW | SUCCEEDED | DRY_RUN | EXISTS_SCAN | RUN_TIMES + null | null | null | null | null | false | 0 + 'sc-123'| null | null | null | null | false | 0 and: - 'sc-123'| 'bd-123' | null | null | false | 0 - 'sc-123'| 'bd-123' | true | null | false | 0 - 'sc-123'| 'bd-123' | false | null | false | 1 - 'sc-123'| 'bd-123' | false | null | true | 0 - 'sc-123'| 'bd-123' | false | true | false | 0 + 'sc-123'| 'bd-123' | null | null | null | false | 0 + 'sc-123'| 'bd-123' | true | null | null | false | 0 + 'sc-123'| 'bd-123' | false | true | null | false | 1 + 'sc-123'| 'bd-123' | false | false | null | false | 0 + 'sc-123'| 'bd-123' | false | null | null | true | 0 + 'sc-123'| 'bd-123' | false | null | true | false | 0 } def 'should store scan entry' () { From d96275a1aced14c77e58c45f4a1f177c67a3db4e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 23 Oct 2024 09:50:34 +0200 Subject: [PATCH 03/54] Fix Do not render inspect url on fail Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/controller/ViewController.groovy | 2 +- src/main/resources/io/seqera/wave/build-view.hbs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 1fafad4b4..69d37d94b 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -221,7 +221,7 @@ class ViewController { binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null binding.scan_id = result.scanId // inspect uri - binding.inspect_url = "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" + binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null // configure build logs when available if( buildLogService ) { final buildLog = buildLogService.fetchLogString(result.buildId) diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 65bb3c9cd..509546fc0 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -125,8 +125,10 @@ Container image {{#if build_in_progress}} {{build_image}} - {{else}} + {{else if inspect_url}} {{build_image}} + {{else}} + {{build_image}} {{/if}} From a958a0624e7e7091f02186b8fe1fa16347c085b3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 23 Oct 2024 09:51:56 +0200 Subject: [PATCH 04/54] [release] bump version 1.13.5 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 80138e714..43ded9062 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.4 +1.13.5 diff --git a/changelog.txt b/changelog.txt index 41b36f097..960bfe42f 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,12 @@ # Wave changelog +1.13.5 - 23 Oct 2024 +- Fix Do not render inspect url on fail [d96275a1] +- Fix inspect view empty nodes (#706) [b3473b7e] +- Fix prevent scan on cached failed builds [4473fe8c] +- Use JedisPool in place of generic connection pool (#711) [cd16cfd1] +- Minor page title change [c3be9304] +- GHA to submit dependency graph to Github (#715) [09c86627] + 1.13.4 - 20 Oct 2024 - Add scan failure duration setting (#705) [372d6dec] - Change scan config log to info [f382c51a] From ab81b6dc01fa00d7f767067fd37dd3cae156d66f Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Fri, 25 Oct 2024 19:27:49 +0530 Subject: [PATCH 05/54] Add scan color for different vuls (#719) Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/controller/ViewController.groovy | 25 +++++++++++++++++++ .../resources/io/seqera/wave/scan-view.hbs | 4 +-- .../wave/controller/ViewControllerTest.groovy | 20 +++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 69d37d94b..a7e88abcb 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.controller import java.util.regex.Pattern +import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value @@ -45,6 +46,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry +import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.JacksonHelper import jakarta.inject.Inject import static io.seqera.wave.util.DataTimeUtils.formatDuration @@ -408,7 +410,10 @@ class ViewController { } Map makeScanViewBinding(WaveScanRecord result, Map binding=new HashMap(10)) { + final color = getScanColor(result.vulnerabilities) binding.should_refresh = !result.done() + binding.scan_color_bg = color.background + binding.scan_color_fg = color.foreground binding.scan_id = result.id binding.scan_container_image = result.containerImage ?: '-' binding.scan_platform = result.platform?.toString() ?: '-' @@ -437,4 +442,24 @@ class ViewController { return binding } + @Canonical + static class Colour { + final background + final foreground + } + + protected static Colour getScanColor(List vulnerabilities){ + boolean hasMedium = vulnerabilities.stream() + .anyMatch(v -> v.severity.equals("MEDIUM")) + boolean hasHighOrCritical = vulnerabilities.stream() + .anyMatch(v -> v.severity.equals("HIGH") || v.severity.equals("CRITICAL")) + if(hasHighOrCritical){ + return new Colour('#ffe4e2', '#e00404') + } + else if(hasMedium){ + return new Colour('#f7dc6f', "#000000") + } + return new Colour('#dff0d8', '#3c763d') + } + } diff --git a/src/main/resources/io/seqera/wave/scan-view.hbs b/src/main/resources/io/seqera/wave/scan-view.hbs index 0ea0ecc1b..66ca925c8 100644 --- a/src/main/resources/io/seqera/wave/scan-view.hbs +++ b/src/main/resources/io/seqera/wave/scan-view.hbs @@ -30,13 +30,13 @@ body { {{#if scan_exist}} {{#if scan_completed}} {{#if scan_failed}} -
+

Unable to complete the container security scan successfully

{{else}} -
+

Container security scan completed

diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index b7dafe358..bbb81e467 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -55,6 +55,9 @@ import io.seqera.wave.tower.User import jakarta.inject.Inject import static io.seqera.wave.util.DataTimeUtils.formatDuration import static io.seqera.wave.util.DataTimeUtils.formatTimestamp + +import static io.seqera.wave.controller.ViewController.Colour + /** * * @author Paolo Di Tommaso @@ -743,4 +746,21 @@ class ViewControllerTest extends Specification { null | false '1234567890abcdef' | true } + + @Unroll + def 'should return correct scan color based on vulnerabilities'() { + expect: + ViewController.getScanColor(VULNERABILITIES) == EXPEXTED_COLOR + + where: + VULNERABILITIES | EXPEXTED_COLOR + [new ScanVulnerability(severity: 'LOW')] | new Colour('#dff0d8','#3c763d') + [new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#f7dc6f','#000000') + [new ScanVulnerability(severity: 'HIGH')] | new Colour('#ffe4e2','#e00404') + [new ScanVulnerability(severity: 'CRITICAL')] | new Colour('#ffe4e2','#e00404') + [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#f7dc6f','#000000') + [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'HIGH')] | new Colour('#ffe4e2','#e00404') + [new ScanVulnerability(severity: 'MEDIUM'), new ScanVulnerability(severity: 'CRITICAL')] | new Colour('#ffe4e2','#e00404') + [] | new Colour('#dff0d8','#3c763d') + } } From 0e2e9f579a98ecf333aa6a4403fe12ca2cbab1a5 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 25 Oct 2024 15:59:51 +0200 Subject: [PATCH 06/54] [release] bump version 1.13.6 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 43ded9062..2e3a551fe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.5 +1.13.6 diff --git a/changelog.txt b/changelog.txt index 960bfe42f..e347c6497 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.13.6 - 25 Oct 2024 +- Add scan color for different vuls (#719) [ab81b6dc] + 1.13.5 - 23 Oct 2024 - Fix Do not render inspect url on fail [d96275a1] - Fix inspect view empty nodes (#706) [b3473b7e] From 0f6003063507e196158ec06ccbf9892feac4ee0d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 25 Oct 2024 17:39:36 +0200 Subject: [PATCH 07/54] Add ability to configure trivy dbs (#720) Signed-off-by: Paolo Di Tommaso --- .../wave/configuration/ScanConfig.groovy | 27 ++++++++---- .../wave/service/k8s/K8sServiceImpl.groovy | 7 +++- .../service/scan/DockerScanStrategy.groovy | 12 +++--- src/main/resources/application-dev.yml | 1 - src/main/resources/application.yml | 6 ++- .../wave/configuration/ScanConfigTest.groovy | 41 +++++++++++++++++++ .../service/k8s/K8sServiceImplTest.groovy | 2 +- .../scan/DockerScanStrategyTest.groovy | 6 ++- 8 files changed, 83 insertions(+), 19 deletions(-) create mode 100644 src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index a0f833f27..1223d137e 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -21,16 +21,14 @@ package io.seqera.wave.configuration import java.nio.file.Files import java.nio.file.Path import java.time.Duration - -import io.micronaut.context.annotation.Requires -import io.micronaut.core.annotation.Nullable import javax.annotation.PostConstruct import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value -import io.seqera.wave.util.StringUtils +import io.micronaut.core.annotation.Nullable import jakarta.inject.Singleton /** * Container Scan service settings @@ -83,8 +81,8 @@ class ScanConfig { Duration scanIdDuration @Nullable - @Value('${wave.scan.github-token}') - String githubToken + @Value('${wave.scan.environment}') + List environment String getScanImage() { return scanImage @@ -118,8 +116,23 @@ class ScanConfig { return severity } + List> getEnvironmentAsTuples() { + if( !environment ) + return List.of() + final result = new ArrayList>() + for( String entry : environment ) { + final p=entry.indexOf('=') + final name = p!=-1 ? entry.substring(0,p) : entry + final value = p!=-1 ? entry.substring(p+1) : '' + if( !value ) + log.warn "Invalid 'wave.scan.environment value' -- offending entry: '$entry'" + result.add(new Tuple2(name,value)) + } + return result + } + @PostConstruct private void init() { - log.info("Scanner config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts; github-token=${StringUtils.redact(githubToken)}") + log.info("Scan config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts; env=${environment}") } } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index ac46c912d..48cecbc30 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -784,8 +784,11 @@ class K8sServiceImpl implements K8sService { .withVolumeMounts(mounts) .withResources(requests) - if( scanConfig.githubToken ) { - container.withEnv(new V1EnvVar().name('GITHUB_TOKEN').value(scanConfig.githubToken)) + final env = scanConfig.environmentAsTuples + for( Tuple2 entry : env ) { + final String k = entry.v1 + final String v = entry.v2 + container.withEnv(new V1EnvVar().name(k).value(v)) } // spec section diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index 430b7ed1d..084e36ec8 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -73,7 +73,7 @@ class DockerScanStrategy extends ScanStrategy { // outfile file name final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) // create the launch command - final dockerCommand = dockerWrapper(jobName, req.workDir, configFile, scanConfig.githubToken) + final dockerCommand = dockerWrapper(jobName, req.workDir, configFile, scanConfig.environment) final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, req.platform, scanConfig) final command = dockerCommand + trivyCommand @@ -89,7 +89,7 @@ class DockerScanStrategy extends ScanStrategy { } } - protected List dockerWrapper(String jobName, Path scanDir, Path credsFile, String githubToken) { + protected List dockerWrapper(String jobName, Path scanDir, Path credsFile, List env) { final wrapper = ['docker','run'] wrapper.add('--detach') @@ -112,9 +112,11 @@ class DockerScanStrategy extends ScanStrategy { wrapper.add("${credsFile}:${Trivy.CONFIG_MOUNT_PATH}:ro".toString()) } - if( githubToken ) { - wrapper.add('-e') - wrapper.add("GITHUB_TOKEN="+githubToken) + if( env ) { + for( String it : env ) { + wrapper.add('-e') + wrapper.add(it) + } } return wrapper diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 7f15e570a..4481f847d 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -4,7 +4,6 @@ wave: enabled: true failure: duration: '30s' - github-token: "${GITHUB_TOKEN:}" build: workspace: 'build-workspace' metrics: diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 38d685608..bd030f589 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -84,7 +84,11 @@ wave: multiplier: '1.75' scan: image: - name: aquasec/trivy:0.55.0 + name: aquasec/trivy:0.56.2 + environment: + # see https://github.com/aquasecurity/trivy/discussions/7668#discussioncomment-11028887 + - "TRIVY_DB_REPOSITORY=public.ecr.aws/aquasecurity/trivy-db" + - "TRIVY_JAVA_DB_REPOSITORY=public.ecr.aws/aquasecurity/trivy-java-db" blobCache: s5cmdImage: public.cr.seqera.io/wave/s5cmd:v2.2.2 --- diff --git a/src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy b/src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy new file mode 100644 index 000000000..c1447b452 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy @@ -0,0 +1,41 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.configuration + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class ScanConfigTest extends Specification { + + def 'should convert env to tuples' () { + given: + def config1 = new ScanConfig() + def config2 = new ScanConfig(environment: ['FOO=one','BAR=two']) + def config3 = new ScanConfig(environment: ['FOO','BAR=']) + + expect: + config1.environmentAsTuples == [] + config2.environmentAsTuples == [new Tuple2('FOO','one'), new Tuple2('BAR','two')] + config3.environmentAsTuples == [new Tuple2('FOO',''), new Tuple2('BAR','')] + + } +} diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 03d136e19..4ebdd7c34 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -748,7 +748,7 @@ class K8sServiceImplTest extends Specification { getCacheDirectory() >> Path.of('/build/cache/dir') getRequestsCpu() >> '2' getRequestsMemory() >> '4Gi' - getGithubToken() >> '123abc' + getEnvironmentAsTuples() >> [new Tuple2('GITHUB_TOKEN', '123abc')] } when: diff --git a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy index f490e0aa1..5e9c898e5 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy @@ -42,7 +42,7 @@ class DockerScanStrategyTest extends Specification { when: def scanDir = Path.of('/some/scan/dir') def config = Path.of("/user/test/build-workspace/config.json") - def command = dockerContainerStrategy.dockerWrapper('foo-123', scanDir, config, 'xyz') + def command = dockerContainerStrategy.dockerWrapper('foo-123', scanDir, config, ['FOO=1', 'BAR=2']) then: command == [ @@ -60,7 +60,9 @@ class DockerScanStrategyTest extends Specification { '-v', '/user/test/build-workspace/config.json:/root/.docker/config.json:ro', '-e', - 'GITHUB_TOKEN=xyz' + 'FOO=1', + '-e', + 'BAR=2' ] cleanup: From aa804e35e75d7efc053882c149040f9f621556e6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 25 Oct 2024 17:41:15 +0200 Subject: [PATCH 08/54] [release] bump version 1.13.7 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2e3a551fe..b0f139ead 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.6 +1.13.7 diff --git a/changelog.txt b/changelog.txt index e347c6497..0a02e82c3 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.13.7 - 25 Oct 2024 +- Add ability to configure trivy environment & DBs (#720) [0f600306] + 1.13.6 - 25 Oct 2024 - Add scan color for different vuls (#719) [ab81b6dc] From f01e4dba274f58ad8eeb6a953b70b13a2ad910eb Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 26 Oct 2024 12:45:44 +0200 Subject: [PATCH 09/54] Improve scan logging Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/service/scan/DockerScanStrategy.groovy | 2 +- .../groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index 084e36ec8..3d340762b 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -53,7 +53,7 @@ class DockerScanStrategy extends ScanStrategy { @Override void scanContainer(String jobName, ScanRequest req) { - log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}") + log.info("Launching container scan job: $jobName for request: ${req}") // create the scan dir try { diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index 136a57a19..34baf87a2 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -65,7 +65,7 @@ class KubeScanStrategy extends ScanStrategy { @Override void scanContainer(String jobName, ScanRequest req) { - log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}") + log.info("Launching container scan job: $jobName for request: ${req}") try{ // create the scan dir try { From e767c367fafe8083b3106dbe10cd88fc3463f07f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 26 Oct 2024 17:02:03 +0200 Subject: [PATCH 10/54] Fix update scan status synchronously Signed-off-by: Paolo Di Tommaso --- .../impl/ContainerBuildServiceImpl.groovy | 33 +++++----- .../mirror/ContainerMirrorServiceImpl.groovy | 9 ++- .../service/scan/ContainerScanService.groovy | 5 +- .../scan/ContainerScanServiceImpl.groovy | 45 ++++++++----- .../builder/ContainerBuildServiceTest.groovy | 64 ++++++++++++------- 5 files changed, 95 insertions(+), 61 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy index 0d36875d5..26250c2c3 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy @@ -29,7 +29,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher import io.micronaut.core.annotation.Nullable -import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.BuildContext import io.seqera.wave.auth.RegistryCredentialsProvider @@ -191,8 +190,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler launch(request), executor) + try { + // create a record to mark the beginning + final scan = ScanEntry.create(request) + if( scanStore.putIfAbsent(scan.scanId, scan) ) { + //start scanning of build container + CompletableFuture.runAsync(() -> launch(request), executor) + } + } + catch (Throwable e){ + log.warn "Unable to save scan result - id=${request.scanId}; cause=${e.message}", e + storeScanEntry(ScanEntry.failure(request)) + } } @Override @@ -181,14 +190,8 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler metricsService.incrementScansCounter(request.identity), executor) - // launch container scan - jobService.launchScan(request) - } + incrScanMetrics(request) + jobService.launchScan(request) } catch (Throwable e){ log.warn "Unable to save scan result - id=${request.scanId}; cause=${e.message}", e @@ -196,6 +199,16 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler) - def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def buildStore = Mock(BuildStateStore) + def proxyService = Mock(RegistryProxyService) + def persistenceService = Mock(PersistenceService) + def scanService = Mock(ContainerScanService) + def eventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, proxyService: proxyService, eventPublisher: eventPublisher, persistenceService: persistenceService, scanService:scanService, buildConfig: buildConfig) def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def state = JobState.succeeded('logs') def res = BuildResult.create('1') @@ -358,7 +361,8 @@ class ContainerBuildServiceTest extends Specification { targetImage: 'docker.io/foo:0', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + identity: PlatformId.NULL ) def build = new BuildEntry(req, res) @@ -366,19 +370,24 @@ class ContainerBuildServiceTest extends Specification { service.onJobCompletion(job, build, state) then: - 1 * mockBuildStore.storeBuild('1', _) + 1 * scanService.scanOnBuild(_) >> null and: - 1 * mockProxyService.getImageDigest(_, _) >> 'digest' + 1 * buildStore.storeBuild(req.targetImage, _) >> null and: - 1 * mockEventPublisher.publishEvent(_) + 1 * proxyService.getImageDigest(_, _) >> 'digest' + and: + 1 * persistenceService.saveBuild(_) >> null + and: + 1 * eventPublisher.publishEvent(_) } def 'should handle job error event and update build store'() { given: - def mockBuildStore = Mock(BuildStateStore) - def mockProxyService = Mock(RegistryProxyService) - def mockEventPublisher = Mock(ApplicationEventPublisher) - def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def buildStore = Mock(BuildStateStore) + def proxyService = Mock(RegistryProxyService) + def persistenceService = Mock(PersistenceService) + def eventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, proxyService: proxyService, eventPublisher: eventPublisher, persistenceService:persistenceService, buildConfig: buildConfig) def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def error = new Exception('error') def res = BuildResult.create('1') @@ -386,7 +395,8 @@ class ContainerBuildServiceTest extends Specification { targetImage: 'docker.io/foo:0', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + identity: PlatformId.NULL ) def build = new BuildEntry(req, res) @@ -394,24 +404,28 @@ class ContainerBuildServiceTest extends Specification { service.onJobException(job, build, error) then: - 1 * mockBuildStore.storeBuild('1', _) + 1 * buildStore.storeBuild(req.targetImage, _) >> null and: - 1 * mockEventPublisher.publishEvent(_) + 1 * persistenceService.saveBuild(_) >> null + and: + 1 * eventPublisher.publishEvent(_) } def 'should handle job timeout event and update build store'() { given: - def mockBuildStore = Mock(BuildStateStore) - def mockProxyService = Mock(RegistryProxyService) - def mockEventPublisher = Mock(ApplicationEventPublisher) - def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def buildStore = Mock(BuildStateStore) + def proxyService = Mock(RegistryProxyService) + def persistenceService = Mock(PersistenceService) + def eventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, proxyService: proxyService, eventPublisher: eventPublisher, persistenceService:persistenceService, buildConfig: buildConfig) def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def res = BuildResult.create('1') def req = new BuildRequest( targetImage: 'docker.io/foo:0', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + identity: PlatformId.NULL ) def build = new BuildEntry(req, res) @@ -419,9 +433,11 @@ class ContainerBuildServiceTest extends Specification { service.onJobTimeout(job, build) then: - 1 * mockBuildStore.storeBuild('1', _) + 1 * buildStore.storeBuild(req.targetImage, _) >> null + and: + 1 * persistenceService.saveBuild(_) >> null and: - 1 * mockEventPublisher.publishEvent(_) + 1 * eventPublisher.publishEvent(_) } } From 705141f086637480151c01809ab01559f9334130 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 26 Oct 2024 17:02:26 +0200 Subject: [PATCH 11/54] Bump scan warn colour Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/controller/ViewController.groovy | 2 +- .../io/seqera/wave/controller/ViewControllerTest.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index a7e88abcb..d30db91d8 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -457,7 +457,7 @@ class ViewController { return new Colour('#ffe4e2', '#e00404') } else if(hasMedium){ - return new Colour('#f7dc6f', "#000000") + return new Colour('#fff8c5', "#000000") } return new Colour('#dff0d8', '#3c763d') } diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index bbb81e467..653be9914 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -755,10 +755,10 @@ class ViewControllerTest extends Specification { where: VULNERABILITIES | EXPEXTED_COLOR [new ScanVulnerability(severity: 'LOW')] | new Colour('#dff0d8','#3c763d') - [new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#f7dc6f','#000000') + [new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#fff8c5','#000000') [new ScanVulnerability(severity: 'HIGH')] | new Colour('#ffe4e2','#e00404') [new ScanVulnerability(severity: 'CRITICAL')] | new Colour('#ffe4e2','#e00404') - [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#f7dc6f','#000000') + [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#fff8c5','#000000') [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'HIGH')] | new Colour('#ffe4e2','#e00404') [new ScanVulnerability(severity: 'MEDIUM'), new ScanVulnerability(severity: 'CRITICAL')] | new Colour('#ffe4e2','#e00404') [] | new Colour('#dff0d8','#3c763d') From 80c0f2789aec8934205fb115ee6a9752dfd1d7ba Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 26 Oct 2024 17:03:45 +0200 Subject: [PATCH 12/54] [release] bump version 1.13.8 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b0f139ead..237a25647 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.7 +1.13.8 diff --git a/changelog.txt b/changelog.txt index 0a02e82c3..69f8fc7e2 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,6 @@ # Wave changelog +1.13.8 - 26 Oct 2024 + 1.13.7 - 25 Oct 2024 - Add ability to configure trivy environment & DBs (#720) [0f600306] From d167b0206309ee9dca8ea765bf40d8471c87e9be Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 28 Oct 2024 08:46:47 +0100 Subject: [PATCH 13/54] Update changelog [ci skip] Signed-off-by: Paolo Di Tommaso --- changelog.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.txt b/changelog.txt index 69f8fc7e2..affdc1112 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,8 @@ # Wave changelog 1.13.8 - 26 Oct 2024 +- Fix update scan status synchronously [e767c367] +- Bump scan warn colour [705141f0] +- Improve scan logging [f01e4dba] 1.13.7 - 25 Oct 2024 - Add ability to configure trivy environment & DBs (#720) [0f600306] From e38e2c446295f538aa939d2bf59720f605e56e28 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Mon, 28 Oct 2024 21:18:24 +0530 Subject: [PATCH 14/54] fixed inspect view (#724) Signed-off-by: munishchouhan --- src/main/resources/io/seqera/wave/inspect-view.hbs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/io/seqera/wave/inspect-view.hbs b/src/main/resources/io/seqera/wave/inspect-view.hbs index 03665443e..68795718d 100644 --- a/src/main/resources/io/seqera/wave/inspect-view.hbs +++ b/src/main/resources/io/seqera/wave/inspect-view.hbs @@ -180,7 +180,7 @@ if (data.hasOwnProperty(key)) { const li = document.createElement('li'); - if (typeof data[key] === 'object' && !Array.isArray(data[key]) && Object.keys(data[key]).length > 0) { + if (typeof data[key] === 'object' && !Array.isArray(data[key]) && Object.keys(data[key]).length > 0 && Object.keys(data[key]).length !== undefined) { const details = document.createElement('details'); const summary = document.createElement('summary'); summary.textContent = key; @@ -210,7 +210,7 @@ details.appendChild(nestedUl); li.appendChild(details); } else { - li.textContent = `${key}: ${data[key]}`; + li.textContent = `${key}: ${JSON.stringify(data[key])}`; } ul.appendChild(li); From dcf41dea974563cbcfc9b4470583ab46e084d639 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Tue, 29 Oct 2024 13:30:32 +0530 Subject: [PATCH 15/54] Fix inspect view (#725) Signed-off-by: munishchouhan --- src/main/resources/io/seqera/wave/inspect-view.hbs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/io/seqera/wave/inspect-view.hbs b/src/main/resources/io/seqera/wave/inspect-view.hbs index 68795718d..030977efe 100644 --- a/src/main/resources/io/seqera/wave/inspect-view.hbs +++ b/src/main/resources/io/seqera/wave/inspect-view.hbs @@ -127,7 +127,6 @@
{{else}} - {{/if}}

Container Specification

@@ -180,7 +179,7 @@ if (data.hasOwnProperty(key)) { const li = document.createElement('li'); - if (typeof data[key] === 'object' && !Array.isArray(data[key]) && Object.keys(data[key]).length > 0 && Object.keys(data[key]).length !== undefined) { + if (typeof data[key] === 'object' && !Array.isArray(data[key]) && Object.keys(data[key]).length > 0) { const details = document.createElement('details'); const summary = document.createElement('summary'); summary.textContent = key; @@ -223,5 +222,6 @@ document.getElementById('config-div').appendChild(createTreeView({{{config}}})); document.getElementById('manifest-div').appendChild(createTreeView({{{manifest}}})); +{{/if}} From cc7b535794c0a4f38f2d6a20c41468e9f08dc8c3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 29 Oct 2024 09:04:53 +0100 Subject: [PATCH 16/54] [release] bump version 1.13.9 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 237a25647..084f22677 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.8 +1.13.9 diff --git a/changelog.txt b/changelog.txt index affdc1112..cb8712b0e 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.13.9 - 29 Oct 2024 +- Fix inspect view (#725) [dcf41dea] [e38e2c44] + 1.13.8 - 26 Oct 2024 - Fix update scan status synchronously [e767c367] - Bump scan warn colour [705141f0] From d42bcae17dd40c90dfa8f4d67f06b890f9c402d8 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 29 Oct 2024 19:23:13 +0100 Subject: [PATCH 17/54] Prevent scan when mode is not defined Signed-off-by: Paolo Di Tommaso --- .../scan/ContainerScanServiceImpl.groovy | 4 +- .../scan/ContainerScanServiceImplTest.groovy | 37 +++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 7ca76aa07..c87020d60 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -132,11 +132,11 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler> SCAN_ID + request.scanMode >> MODE request.isContainer() >> CONTAINER request.dryRun >> DRY_RUN and: @@ -379,12 +381,14 @@ class ContainerScanServiceImplTest extends Specification { RUN_TIMES * scanService.scan(scan) >> null where: - SCAN_ID | CONTAINER | DRY_RUN | RUN_TIMES - null | false | false | 0 - 'sc-123'| false | false | 0 - 'sc-123'| true | false | 1 - 'sc-123'| true | true | 0 - null | true | false | 0 + SCAN_ID | MODE | CONTAINER | DRY_RUN | RUN_TIMES + null | ScanMode.async | false | false | 0 + 'sc-123'| ScanMode.async | false | false | 0 + 'sc-123'| ScanMode.async | true | false | 1 + 'sc-123'| ScanMode.required | true | false | 1 + 'sc-123'| ScanMode.none | true | false | 0 + 'sc-123'| ScanMode.async | true | true | 0 + null | ScanMode.async | true | false | 0 } @@ -398,6 +402,7 @@ class ContainerScanServiceImplTest extends Specification { scanService.existsScan(SCAN_ID) >> EXISTS_SCAN and: def request = Mock(ContainerRequest) + request.scanMode >> MODE request.scanId >> SCAN_ID request.buildId >> BUILD_ID request.buildNew >> BUILD_NEW @@ -413,16 +418,18 @@ class ContainerScanServiceImplTest extends Specification { RUN_TIMES * scanService.scan(scan) >> null where: - SCAN_ID | BUILD_ID | BUILD_NEW | SUCCEEDED | DRY_RUN | EXISTS_SCAN | RUN_TIMES - null | null | null | null | null | false | 0 - 'sc-123'| null | null | null | null | false | 0 + SCAN_ID | BUILD_ID | BUILD_NEW | SUCCEEDED | MODE | DRY_RUN | EXISTS_SCAN | RUN_TIMES + null | null | null | null | ScanMode.async | null | false | 0 + 'sc-123'| null | null | null | ScanMode.async | null | false | 0 and: - 'sc-123'| 'bd-123' | null | null | null | false | 0 - 'sc-123'| 'bd-123' | true | null | null | false | 0 - 'sc-123'| 'bd-123' | false | true | null | false | 1 - 'sc-123'| 'bd-123' | false | false | null | false | 0 - 'sc-123'| 'bd-123' | false | null | null | true | 0 - 'sc-123'| 'bd-123' | false | null | true | false | 0 + 'sc-123'| 'bd-123' | null | null | ScanMode.async | null | false | 0 + 'sc-123'| 'bd-123' | true | null | ScanMode.async | null | false | 0 + 'sc-123'| 'bd-123' | false | true | ScanMode.async | null | false | 1 + 'sc-123'| 'bd-123' | false | true | ScanMode.required | null | false | 1 + 'sc-123'| 'bd-123' | false | true | ScanMode.none | null | false | 0 + 'sc-123'| 'bd-123' | false | false | ScanMode.async | null | false | 0 + 'sc-123'| 'bd-123' | false | null | ScanMode.async | null | true | 0 + 'sc-123'| 'bd-123' | false | null | ScanMode.async | true | false | 0 } def 'should store scan entry' () { From e8a6b7eeabe52cc1fa6079a15bdc268552dca7b0 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 29 Oct 2024 21:11:01 +0100 Subject: [PATCH 18/54] Log slow processing stream messages Signed-off-by: Paolo Di Tommaso --- .../data/stream/impl/RedisMessageStream.groovy | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy index 609b71061..c417568b3 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy @@ -62,6 +62,9 @@ class RedisMessageStream implements MessageStream { @Value('${wave.message-stream.claim-timeout:5s}') private Duration claimTimeout + @Value('${wave.message-stream.consume-warn-timeout-millis:4000}') + private long consumeWarnTimeoutMillis + private String consumerName @PostConstruct @@ -102,11 +105,17 @@ class RedisMessageStream implements MessageStream { @Override boolean consume(String streamId, MessageConsumer consumer) { try (Jedis jedis = pool.getResource()) { + String msg + final long begin = System.currentTimeMillis() final entry = claimMessage(jedis,streamId) ?: readMessage(jedis, streamId) - if( entry && consumer.accept(entry.getFields().get(DATA_FIELD)) ) { + if( entry && consumer.accept(msg=entry.getFields().get(DATA_FIELD)) ) { final tx = jedis.multi() // acknowledge the entry has been processed so that it cannot be claimed anymore tx.xack(streamId, CONSUMER_GROUP_NAME, entry.getID()) + final delta = System.currentTimeMillis()-begin + if( delta>consumeWarnTimeoutMillis ) { + log.warn "Redis message stream - consume processing took ${Duration.ofMillis(delta)} - offending entry=${entry.getID()}; message=${msg}" + } // this remove permanently the entry from the stream tx.xdel(streamId, entry.getID()) tx.exec() From d1d1a6228bac8013632742bfa6654438f1f9e1d9 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 29 Oct 2024 21:12:54 +0100 Subject: [PATCH 19/54] [release] bump version 1.13.10 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 084f22677..f3352b3fe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.9 +1.13.10 diff --git a/changelog.txt b/changelog.txt index cb8712b0e..36eae4524 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,8 @@ # Wave changelog +1.13.10 - 29 Oct 2024 +- Log slow processing stream messages [e8a6b7ee] +- Prevent scan when mode is not defined [d42bcae1] + 1.13.9 - 29 Oct 2024 - Fix inspect view (#725) [dcf41dea] [e38e2c44] From 5b4e5f0f1a8ad2b4978f2e5b9421a0db3508ae7c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 31 Oct 2024 17:37:03 +0100 Subject: [PATCH 20/54] Update CI secrets Signed-off-by: Paolo Di Tommaso --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 503bce24f..993808b58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.TOWER_CI_AWS_SECRET}} DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PAT: ${{ secrets.DOCKER_PAT }} - QUAY_USER: ${{ secrets.QUAY_USER }} + QUAY_USER: "pditommaso+wave_ci_tests" QUAY_PAT: ${{ secrets.QUAY_PAT }} AZURECR_USER: ${{ secrets.AZURECR_USER }} AZURECR_PAT: ${{ secrets.AZURECR_PAT }} From adb750073ac2af94dc9f2396f9bfdfc6ae7f5492 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Fri, 1 Nov 2024 12:56:22 +0530 Subject: [PATCH 21/54] Bump org.apache.commons:commons-compress:1.27.1 (#722) Signed-off-by: munishchouhan Co-authored-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 28442ab3d..02834ad7d 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,7 @@ dependencies { implementation 'dev.failsafe:failsafe:3.1.0' implementation('io.projectreactor:reactor-core') implementation("io.seqera:tower-crypto:22.4.0-watson") { transitive = false } // to be replaced with 22.4.0 once released - implementation 'org.apache.commons:commons-compress:1.24.0' + implementation 'org.apache.commons:commons-compress:1.27.1' implementation 'org.apache.commons:commons-lang3:3.12.0' implementation 'io.kubernetes:client-java:19.0.0' implementation 'io.kubernetes:client-java-api-fluent:18.0.1' From 2f0d8f9f24ea1edebf42d52ef7d33778a4b736d6 Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Sat, 2 Nov 2024 03:11:44 +0530 Subject: [PATCH 22/54] Cap number of scan vulnerabilities reported (#728) Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../wave/configuration/ScanConfig.groovy | 5 +- .../scan/ContainerScanServiceImpl.groovy | 6 +- .../service/scan/ScanVulnerability.groovy | 2 +- .../service/scan/TrivyResultProcessor.groovy | 21 ++++- .../scan/ContainerScanServiceImplTest.groovy | 2 +- .../scan/TrivyResultProcessorTest.groovy | 89 ++++++++++++++++++- 6 files changed, 114 insertions(+), 11 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index 1223d137e..a3232f300 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -84,6 +84,9 @@ class ScanConfig { @Value('${wave.scan.environment}') List environment + @Value('${wave.scan.vulnerability.limit:100}') + Integer vulnerabilityLimit + String getScanImage() { return scanImage } @@ -133,6 +136,6 @@ class ScanConfig { @PostConstruct private void init() { - log.info("Scan config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts; env=${environment}") + log.info("Scan config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; vulnerability-limit: $vulnerabilityLimit; retry-attempts: $retryAttempts; env=${environment}") } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index c87020d60..1571bb50d 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -269,12 +269,14 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler { - static final Map ORDER = Map.of( + static final private Map ORDER = Map.of( 'LOW', 0, 'MEDIUM', 1, 'HIGH', 2, diff --git a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy index a1055037b..a4bd14788 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy @@ -21,6 +21,8 @@ package io.seqera.wave.service.scan import java.nio.file.Path import groovy.json.JsonSlurper +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.exception.ScanRuntimeException /** @@ -30,16 +32,23 @@ import io.seqera.wave.exception.ScanRuntimeException * @author Paolo Di Tommaso */ @Slf4j +@CompileStatic class TrivyResultProcessor { - static List process(Path reportFile) { - process(reportFile.getText()) + static List parse(Path scanFile) { + return parse(scanFile.getText()) } - static List process(String trivyResult) { + static List parse(Path scanFile, int maxEntries) { + final result = parse(scanFile) + return filter(result, maxEntries) + } + + @CompileDynamic + static List parse(String scanJson) { final slurper = new JsonSlurper() try{ - final jsonMap = slurper.parseText(trivyResult) as Map + final jsonMap = slurper.parseText(scanJson) as Map return jsonMap.Results.collect { result -> result.Vulnerabilities.collect { vulnerability -> new ScanVulnerability( @@ -57,4 +66,8 @@ class TrivyResultProcessor { throw new ScanRuntimeException("Failed to parse the trivy result", e) } } + + static protected List filter(List vulnerabilities, int limit){ + vulnerabilities.toSorted((v,w) -> w.compareTo(v)).take(limit) + } } diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index 2410142c1..4ab8ca50b 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -151,7 +151,7 @@ class ContainerScanServiceImplTest extends Specification { and: def KEY = 'scan-20' def jobService = Mock(JobService) - def service = new ContainerScanServiceImpl(scanStore: scanStore, persistenceService: persistenceService, jobService: jobService) + def service = new ContainerScanServiceImpl(scanStore: scanStore, persistenceService: persistenceService, jobService: jobService, config: new ScanConfig(vulnerabilityLimit: 100)) def job = JobSpec.scan(KEY, 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), workDir) def scan = ScanEntry.of(scanId: KEY, buildId: 'build-20', containerImage: 'ubuntu:latest', startTime: Instant.now()) diff --git a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy index 0563f8842..8bb0186b6 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy @@ -91,7 +91,7 @@ class TrivyResultProcessorTest extends Specification { """ when: - def result = TrivyResultProcessor.process(trivyDockerResulJson) + def result = TrivyResultProcessor.parse(trivyDockerResulJson) then: def vulnerability = result[0] @@ -105,9 +105,94 @@ class TrivyResultProcessorTest extends Specification { } + def "should return a sorted map of vulnerabilities"() { + given: + def trivyDockerResulJson = """ + { "Results": [ + { + "Target": "sample-application", + "Class": "os-pkgs", + "Type": "linux", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2023-0001", + "PkgID": "example-lib@1.0.0", + "PkgName": "example-lib", + "InstalledVersion": "1.0.0", + "FixedVersion": "1.0.1", + "Severity": "LOW", + "Description": "A minor vulnerability with low impact.", + "PrimaryURL": "https://example.com/CVE-2023-0001" + }, + { + "VulnerabilityID": "CVE-2023-0002", + "PkgID": "example-lib@1.2.3", + "PkgName": "example-lib", + "InstalledVersion": "1.2.3", + "FixedVersion": "1.2.4", + "Severity": "MEDIUM", + "Description": "A vulnerability that allows unauthorized access.", + "PrimaryURL": "https://example.com/CVE-2023-0002" + }, + { + "VulnerabilityID": "CVE-2023-0003", + "PkgID": "example-lib@2.3.4", + "PkgName": "example-lib", + "InstalledVersion": "2.3.4", + "FixedVersion": "2.3.5", + "Severity": "HIGH", + "Description": "A vulnerability that could lead to remote code execution.", + "PrimaryURL": "https://example.com/CVE-2023-0003" + }, + { + "VulnerabilityID": "CVE-2023-0004", + "PkgID": "example-lib@3.0.0", + "PkgName": "example-lib", + "InstalledVersion": "3.0.0", + "FixedVersion": "3.0.1", + "Severity": "HIGH", + "Description": "A random test vulnerability with unspecified impact.", + "PrimaryURL": "https://example.com/CVE-2023-0004" + }, + { + "VulnerabilityID": "CVE-2023-0005", + "PkgID": "example-lib@3.1.0", + "PkgName": "example-lib", + "InstalledVersion": "3.1.0", + "FixedVersion": "3.1.1", + "Severity": "CRITICAL", + "Description": "Another random test vulnerability for testing purposes.", + "PrimaryURL": "https://example.com/CVE-2023-0005" + } + ] + } + ] + }""".stripIndent() + + when: + def result = TrivyResultProcessor.parse(trivyDockerResulJson) + result = TrivyResultProcessor.filter(result, 4) + + then: + result.size() == 4 + result[0].severity == "CRITICAL" + result[0].id == "CVE-2023-0005" + result[1].severity == "HIGH" + result[1].id == "CVE-2023-0003" + result[2].severity == "HIGH" + result[2].id == "CVE-2023-0004" + result[3].severity == "MEDIUM" + result[3].id == "CVE-2023-0002" + } + + def 'should not fail with empty list' () { + expect: + TrivyResultProcessor.filter([], 10) == [] + } + def "process should throw exception if json is not correct"() { when: - TrivyResultProcessor.process("invalid json") + TrivyResultProcessor.parse("invalid json") then: thrown ScanRuntimeException } From 3ad82a3a08ba70aed33bd8b6db25d6f89648071e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 09:13:45 +0100 Subject: [PATCH 23/54] Save scan record async (#730) Signed-off-by: Paolo Di Tommaso --- .../wave/configuration/ScanConfig.groovy | 1 + .../persistence/impl/SurrealClient.groovy | 6 +++ .../impl/SurrealPersistenceService.groovy | 31 ++++++++++----- .../scan/ContainerScanServiceImpl.groovy | 2 +- .../service/scan/TrivyResultProcessor.groovy | 24 ++++++++---- .../impl/SurrealPersistenceServiceTest.groovy | 6 ++- .../scan/TrivyResultProcessorTest.groovy | 38 +++++++++++-------- 7 files changed, 72 insertions(+), 36 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index a3232f300..baef40d75 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -84,6 +84,7 @@ class ScanConfig { @Value('${wave.scan.environment}') List environment + @Nullable @Value('${wave.scan.vulnerability.limit:100}') Integer vulnerabilityLimit diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy index badd7c4aa..2901ec22f 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy @@ -54,9 +54,15 @@ interface SurrealClient { @Post("/sql") Flux> sqlAsync(@Header String authorization, @Body String body) + @Post("/sql") + Flux>> sqlAsyncMany(@Header String authorization, @Body String body) + @Post("/sql") Map sqlAsMap(@Header String authorization, @Body String body) + @Post("/sql") + List> sqlAsList(@Header String authorization, @Body String body) + @Post("/sql") String sqlAsString(@Header String authorization, @Body String body) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 74079f1b7..64ad6f7c8 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -256,25 +256,36 @@ class SurrealPersistenceService implements PersistenceService { void saveScanRecord(WaveScanRecord scanRecord) { final vulnerabilities = scanRecord.vulnerabilities ?: List.of() + // create a multi-command surreal sql statement to insert all vulnerabilities + // and the scan record in a single operation + List ids = new ArrayList<>(101) + String statement = '' // save all vulnerabilities for( ScanVulnerability it : vulnerabilities ) { - surrealDb.insertScanVulnerability(authorization, it) + statement += "INSERT INTO wave_scan_vuln ${JacksonHelper.toJson(it)};\n" + ids << "wave_scan_vuln:⟨$it.id⟩".toString() } - // compose the list of ids - final ids = vulnerabilities - .collect(it-> "wave_scan_vuln:⟨$it.id⟩".toString()) - - // scan object final copy = scanRecord.clone() copy.vulnerabilities = List.of() final json = JacksonHelper.toJson(copy) - // create the scan record - final statement = "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)}".toString() - final result = surrealDb.sqlAsMap(authorization, statement) - log.trace "Scan update result=$result" + // add the wave_scan record + statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString() + // store the statement using an async operation + surrealDb + .sqlAsyncMany(getAuthorization(), statement) + .subscribe({result -> + log.trace "Scan record save result=$result" + }, + {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error saving scan record => ${msg}\n", error) + }) } protected String patchScanVulnerabilities(String json, List ids) { diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 1571bb50d..73305463e 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -270,7 +270,7 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler parse(Path scanFile) { - return parse(scanFile.getText()) - } - - static List parse(Path scanFile, int maxEntries) { - final result = parse(scanFile) - return filter(result, maxEntries) + /** + * Parse a Trivy vulnerabilities JSON file and return a list of {@link ScanVulnerability} entries + * + * @param scanFile + * The {@link Path} of the Trivy JSON file to be scanned + * @param maxEntries + * The max number of vulnerabilities that should be returned giving precedence to the + * most severe vulnerabilities e.g. one critical and one medium issues are found and + * 1 is specified as {@code maxEntries} only the critical issues is returned. + * @return + * The list of {@link ScanVulnerability} entries as parsed in from the JSON file. + */ + static List parseFile(Path scanFile, Integer maxEntries=null) { + final result = parseJson(scanFile.getText()) + return maxEntries>0 ? filter(result, maxEntries) : result } @CompileDynamic - static List parse(String scanJson) { + static List parseJson(String scanJson) { final slurper = new JsonSlurper() try{ final jsonMap = slurper.parseText(scanJson) as Map diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index ea27f24b7..43227ad5c 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -324,6 +324,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null) when: persistence.saveScanRecord(scan) + sleep 200 then: def result = persistence.loadScanRecord(SCAN_ID) and: @@ -343,6 +344,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe and: // should save the same CVE into another build persistence.saveScanRecord(scanRecord2) + sleep 200 then: def result2 = persistence.loadScanRecord(SCAN_ID2) and: @@ -372,6 +374,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe when: persistence.saveScanRecord(scan) + sleep 200 then: persistence.existsScanRecord(SCAN_ID) } @@ -566,7 +569,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe persistence.saveScanRecord(scan2) persistence.saveScanRecord(scan3) persistence.saveScanRecord(scan4) - + and: + sleep 200 then: persistence.allScans("1234567890abcdef") == [scan3, scan2, scan1] and: diff --git a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy index 8bb0186b6..af9b4b549 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy @@ -18,11 +18,11 @@ package io.seqera.wave.service.scan - import spock.lang.Specification -import io.seqera.wave.exception.ScanRuntimeException +import java.nio.file.Files +import io.seqera.wave.exception.ScanRuntimeException /** * * @author Munish Chouhan @@ -91,7 +91,7 @@ class TrivyResultProcessorTest extends Specification { """ when: - def result = TrivyResultProcessor.parse(trivyDockerResulJson) + def result = TrivyResultProcessor.parseJson(trivyDockerResulJson) then: def vulnerability = result[0] @@ -107,7 +107,10 @@ class TrivyResultProcessorTest extends Specification { def "should return a sorted map of vulnerabilities"() { given: - def trivyDockerResulJson = """ + def folder = Files.createTempDirectory('test') + def scan = folder.resolve('scan.json') + and: + scan.text = """ { "Results": [ { "Target": "sample-application", @@ -170,19 +173,22 @@ class TrivyResultProcessorTest extends Specification { }""".stripIndent() when: - def result = TrivyResultProcessor.parse(trivyDockerResulJson) - result = TrivyResultProcessor.filter(result, 4) + def topIssues = TrivyResultProcessor.parseFile(scan, 2) then: - result.size() == 4 - result[0].severity == "CRITICAL" - result[0].id == "CVE-2023-0005" - result[1].severity == "HIGH" - result[1].id == "CVE-2023-0003" - result[2].severity == "HIGH" - result[2].id == "CVE-2023-0004" - result[3].severity == "MEDIUM" - result[3].id == "CVE-2023-0002" + topIssues.size() == 2 + topIssues[0].severity == "CRITICAL" + topIssues[0].id == "CVE-2023-0005" + topIssues[1].severity == "HIGH" + topIssues[1].id == "CVE-2023-0003" + + when: + def allIssues = TrivyResultProcessor.parseFile(scan) + then: + allIssues.size() == 5 + + cleanup: + folder?.deleteDir() } def 'should not fail with empty list' () { @@ -192,7 +198,7 @@ class TrivyResultProcessorTest extends Specification { def "process should throw exception if json is not correct"() { when: - TrivyResultProcessor.parse("invalid json") + TrivyResultProcessor.parseJson("invalid json") then: thrown ScanRuntimeException } From 38114d7550190f096a7ff0b354b3c227bff68110 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 09:25:42 +0100 Subject: [PATCH 24/54] Rename async methods for semantic consistency Signed-off-by: Paolo Di Tommaso --- .../controller/ContainerController.groovy | 2 +- .../wave/core/RegistryProxyService.groovy | 2 +- .../impl/ContainerBuildServiceImpl.groovy | 2 +- .../mirror/ContainerMirrorServiceImpl.groovy | 6 ++-- .../persistence/PersistenceService.groovy | 10 +++--- .../impl/LocalPersistenceService.groovy | 10 +++--- .../impl/SurrealPersistenceService.groovy | 10 +++--- .../scan/ContainerScanServiceImpl.groovy | 2 +- .../controller/BuildControllerTest.groovy | 4 +-- .../wave/controller/ScanControllerTest.groovy | 2 +- .../wave/controller/ViewControllerTest.groovy | 22 ++++++------ .../builder/ContainerBuildServiceTest.groovy | 6 ++-- .../mirror/ContainerMirrorServiceTest.groovy | 2 +- .../impl/SurrealPersistenceServiceTest.groovy | 34 +++++++++---------- .../scan/ContainerScanServiceImplTest.groovy | 6 ++-- 15 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 713ffae89..fabef4017 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -284,7 +284,7 @@ class ContainerController { protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequest data, TokenData token, String target, String ip) { try { final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration) - persistenceService.saveContainerRequest(recrd) + persistenceService.saveContainerRequestAsync(recrd) } catch (Throwable e) { log.error("Unable to store container request with token: ${token}", e) diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index 1d9bf61de..32cf49604 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -133,7 +133,7 @@ class RegistryProxyService { return try { - persistenceService.updateContainerRequest(route.token, digest) + persistenceService.updateContainerRequestAsync(route.token, digest) } catch (Throwable t) { log.error("Unable store container request for token: $route.token", t) } diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy index 26250c2c3..6dbe9f787 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy @@ -352,7 +352,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler mirrorStore = new HashMap<>() @Override - void saveBuild(WaveBuildRecord record) { + void saveBuildAsync(WaveBuildRecord record) { buildStore[record.buildId] = record } @@ -76,12 +76,12 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveContainerRequest(WaveContainerRecord data) { + void saveContainerRequestAsync(WaveContainerRecord data) { requestStore.put(data.id, data) } @Override - void updateContainerRequest(String token, ContainerDigestPair digest) { + void updateContainerRequestAsync(String token, ContainerDigestPair digest) { final data = requestStore.get(token) if( data ) { requestStore.put(token, new WaveContainerRecord(data, digest.source, digest.target)) @@ -99,7 +99,7 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveScanRecord(WaveScanRecord scanRecord) { + void saveScanRecordAsync(WaveScanRecord scanRecord) { scanStore.put(scanRecord.id, scanRecord) } @@ -129,7 +129,7 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveMirrorResult(MirrorResult mirror) { + void saveMirrorResultAsync(MirrorResult mirror) { mirrorStore.put(mirror.mirrorId, mirror) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 64ad6f7c8..829c6a384 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -100,7 +100,7 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveBuild(WaveBuildRecord build) { + void saveBuildAsync(WaveBuildRecord build) { // note: use surreal sql in order to by-pass issue with large payload // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 final query = "INSERT INTO wave_build ${JacksonHelper.toJson(build)}" @@ -198,7 +198,7 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveContainerRequest(WaveContainerRecord data) { + void saveContainerRequestAsync(WaveContainerRecord data) { // note: use surreal sql in order to by-pass issue with large payload // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 final query = "INSERT INTO wave_request ${JacksonHelper.toJson(data)}" @@ -216,7 +216,7 @@ class SurrealPersistenceService implements PersistenceService { }) } - void updateContainerRequest(String token, ContainerDigestPair digest) { + void updateContainerRequestAsync(String token, ContainerDigestPair digest) { final query = """\ UPDATE wave_request:$token SET sourceDigest = '$digest.source', @@ -253,7 +253,7 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveScanRecord(WaveScanRecord scanRecord) { + void saveScanRecordAsync(WaveScanRecord scanRecord) { final vulnerabilities = scanRecord.vulnerabilities ?: List.of() // create a multi-command surreal sql statement to insert all vulnerabilities @@ -380,7 +380,7 @@ class SurrealPersistenceService implements PersistenceService { * @param mirror {@link MirrorEntry} object */ @Override - void saveMirrorResult(MirrorResult mirror) { + void saveMirrorResultAsync(MirrorResult mirror) { surrealDb.insertMirrorAsync(getAuthorization(), mirror).subscribe({ result-> log.trace "Mirror request with id '$mirror.mirrorId' saved record: ${result}" }, {error-> diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 73305463e..d0e3d7511 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -304,7 +304,7 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler> 'digest' and: - 1 * persistenceService.saveBuild(_) >> null + 1 * persistenceService.saveBuildAsync(_) >> null and: 1 * eventPublisher.publishEvent(_) } @@ -406,7 +406,7 @@ class ContainerBuildServiceTest extends Specification { then: 1 * buildStore.storeBuild(req.targetImage, _) >> null and: - 1 * persistenceService.saveBuild(_) >> null + 1 * persistenceService.saveBuildAsync(_) >> null and: 1 * eventPublisher.publishEvent(_) } @@ -435,7 +435,7 @@ class ContainerBuildServiceTest extends Specification { then: 1 * buildStore.storeBuild(req.targetImage, _) >> null and: - 1 * persistenceService.saveBuild(_) >> null + 1 * persistenceService.saveBuildAsync(_) >> null and: 1 * eventPublisher.publishEvent(_) } diff --git a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy index d892af4f6..c9eedc079 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy @@ -105,7 +105,7 @@ class ContainerMirrorServiceTest extends Specification { and: def state = MirrorResult.of(request) and: - persistenceService.saveMirrorResult(state) + persistenceService.saveMirrorResultAsync(state) when: def copy = mirrorService.getMirrorResult(request.mirrorId) then: diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index 43227ad5c..a8eb7c47b 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -128,7 +128,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe when: storage.initializeDb() and: - storage.saveBuild(build) + storage.saveBuildAsync(build) then: sleep 100 def stored = storage.loadBuild(request.buildId) @@ -160,7 +160,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def record = WaveBuildRecord.fromEvent(event) and: - persistence.saveBuild(record) + persistence.saveBuildAsync(record) when: sleep 100 @@ -248,7 +248,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) when: - persistence.saveBuild(build1) + persistence.saveBuildAsync(build1) sleep 100 then: persistence.loadBuild(request.buildId) == build1 @@ -281,7 +281,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe and: def request = new WaveContainerRecord(req, data, wave, addr, exp) and: - persistence.saveContainerRequest(request) + persistence.saveContainerRequestAsync(request) and: sleep 200 // <-- the above request is async, give time to save it @@ -293,7 +293,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe // should update the record when: - persistence.updateContainerRequest(TOKEN, new ContainerDigestPair('111', '222')) + persistence.updateContainerRequestAsync(TOKEN, new ContainerDigestPair('111', '222')) and: sleep 200 then: @@ -323,7 +323,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null) when: - persistence.saveScanRecord(scan) + persistence.saveScanRecordAsync(scan) sleep 200 then: def result = persistence.loadScanRecord(SCAN_ID) @@ -343,7 +343,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1, CVE4], 1, "Error 'quote'") and: // should save the same CVE into another build - persistence.saveScanRecord(scanRecord2) + persistence.saveScanRecordAsync(scanRecord2) sleep 200 then: def result2 = persistence.loadScanRecord(SCAN_ID2) @@ -373,7 +373,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe !persistence.existsScanRecord(SCAN_ID) when: - persistence.saveScanRecord(scan) + persistence.saveScanRecordAsync(scan) sleep 200 then: persistence.existsScanRecord(SCAN_ID) @@ -401,7 +401,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe storage.initializeDb() and: def result = MirrorEntry.of(request).getResult() - storage.saveMirrorResult(result) + storage.saveMirrorResultAsync(result) sleep 100 when: @@ -461,9 +461,9 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def result1 = MirrorResult.of(request1).complete(1, 'err') def result2 = MirrorResult.of(request2).complete(0, 'ok') def result3 = MirrorResult.of(request3).complete(0, 'ok') - storage.saveMirrorResult(result1) - storage.saveMirrorResult(result2) - storage.saveMirrorResult(result3) + storage.saveMirrorResultAsync(result1) + storage.saveMirrorResultAsync(result2) + storage.saveMirrorResultAsync(result3) sleep 100 when: @@ -509,7 +509,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) when: - persistence.saveBuild(build1) + persistence.saveBuildAsync(build1) sleep 100 then: persistence.loadBuild(request.buildId) == build1 @@ -565,10 +565,10 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def scan4 = new WaveScanRecord('sc-01234567890abcdef_4', '103', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null) when: - persistence.saveScanRecord(scan1) - persistence.saveScanRecord(scan2) - persistence.saveScanRecord(scan3) - persistence.saveScanRecord(scan4) + persistence.saveScanRecordAsync(scan1) + persistence.saveScanRecordAsync(scan2) + persistence.saveScanRecordAsync(scan3) + persistence.saveScanRecordAsync(scan4) and: sleep 200 then: diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index 4ab8ca50b..05fe4efac 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -452,21 +452,21 @@ class ContainerScanServiceImplTest extends Specification { scanService.storeScanEntry(scanSucceeded) then: 1 * scanStore.storeScan(scanSucceeded) >> null - 1 * persistenceService.saveScanRecord(new WaveScanRecord(scanSucceeded)) >> null + 1 * persistenceService.saveScanRecordAsync(new WaveScanRecord(scanSucceeded)) >> null 0 * cleanupService.cleanupScanId(container) >> null when: scanService.storeScanEntry(scanNotDone) then: 1 * scanStore.storeScan(scanNotDone) >> null - 1 * persistenceService.saveScanRecord(new WaveScanRecord(scanNotDone)) >> null + 1 * persistenceService.saveScanRecordAsync(new WaveScanRecord(scanNotDone)) >> null 0 * cleanupService.cleanupScanId(container) >> null when: scanService.storeScanEntry(scanFailed) then: 1 * scanStore.storeScan(scanFailed) >> null - 1 * persistenceService.saveScanRecord(new WaveScanRecord(scanFailed)) >> null + 1 * persistenceService.saveScanRecordAsync(new WaveScanRecord(scanFailed)) >> null 1 * cleanupService.cleanupScanId(container) >> null } } From 2c5d8e031520e8c59d715de474c6f2b80b0edc57 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 09:27:50 +0100 Subject: [PATCH 25/54] [release] bump version 1.13.11 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f3352b3fe..66fae2a63 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.10 +1.13.11 diff --git a/changelog.txt b/changelog.txt index 36eae4524..3085abccb 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,10 @@ # Wave changelog +1.13.11 - 2 Nov 2024 +- Rename async methods for semantic consistency [38114d75] +- Save scan record async (#730) [3ad82a3a] +- Cap number of vulnerabilities reported in scan report to 100 (#728) [2f0d8f9f] +- Bump org.apache.commons:commons-compress:1.27.1 (#722) [adb75007] + 1.13.10 - 29 Oct 2024 - Log slow processing stream messages [e8a6b7ee] - Prevent scan when mode is not defined [d42bcae1] From 1502696da0de271496f83cf485bb4a88a59d72fb Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 11:46:28 +0100 Subject: [PATCH 26/54] Bump spillway 3.0.0 (#731) Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 02834ad7d..740119658 100644 --- a/build.gradle +++ b/build.gradle @@ -88,7 +88,7 @@ dependencies { implementation("ch.qos.logback:logback-classic:1.4.8") // rate limit - implementation 'com.github.seqeralabs:spillway:7b72700293' + implementation 'com.coveo:spillway:3.0.0' // monitoring implementation "io.micronaut.micrometer:micronaut-micrometer-registry-prometheus" From 481298bf74c32dc9e94f73b0bd6fca9176898cd8 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 15:04:04 +0100 Subject: [PATCH 27/54] Remove deprecated K8s methods (#734) Signed-off-by: Paolo Di Tommaso --- .../seqera/wave/service/k8s/K8sService.groovy | 24 +-- .../wave/service/k8s/K8sServiceImpl.groovy | 181 +----------------- .../blob/impl/KubeTransferStrategyTest.groovy | 4 - .../service/k8s/K8sServiceImplTest.groovy | 55 +----- 4 files changed, 3 insertions(+), 261 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 0d8de9015..ccaedec07 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -23,11 +23,9 @@ import java.time.Duration import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.MirrorConfig - +import io.seqera.wave.configuration.ScanConfig /** * Defines Kubernetes operations * @@ -43,23 +41,6 @@ interface K8sService { void deletePod(String name) - @Deprecated - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) - - @Deprecated - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) - - @Deprecated - Integer waitPodCompletion(V1Pod pod, long timeout) - - @Deprecated - void deletePodWhenReachStatus(String podName, String statusName, long timeout) - - @Deprecated - V1Job createJob(String name, String containerImage, List args) - - V1Job getJob(String name) - JobStatus getJobStatus(String name) void deleteJob(String name) @@ -72,9 +53,6 @@ interface K8sService { V1Job launchMirrorJob(String name, String containerImage, List args, Path workDir, Path creds, MirrorConfig config) - @Deprecated - V1PodList waitJob(V1Job job, Long timeout) - V1Pod getLatestPodForJob(String jobName) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 48cecbc30..bd88f3974 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -22,7 +22,6 @@ import java.nio.file.Path import java.time.Duration import javax.annotation.PostConstruct -import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.kubernetes.client.custom.Quantity @@ -36,7 +35,6 @@ import io.kubernetes.client.openapi.models.V1JobStatus import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodBuilder -import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1ResourceRequirements import io.kubernetes.client.openapi.models.V1Volume import io.kubernetes.client.openapi.models.V1VolumeMount @@ -46,9 +44,9 @@ import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton @@ -136,61 +134,6 @@ class K8sServiceImpl implements K8sService { } } - /** - * Create a K8s job with the specified name - * - * @param name - * The K8s job name. It must be unique - * @param containerImage - * The container image to be used to run the job - * @param args - * The command to be executed by the job - * @return - * An instance of {@link V1Job} - */ - @Override - @CompileDynamic - @Deprecated - V1Job createJob(String name, String containerImage, List args) { - - V1Job body = new V1JobBuilder() - .withNewMetadata() - .withNamespace(namespace) - .withName(name) - .endMetadata() - .withNewSpec() - .withBackoffLimit(0) - .withNewTemplate() - .editOrNewSpec() - .addNewContainer() - .withName(name) - .withImage(containerImage) - .withArgs(args) - .endContainer() - .withRestartPolicy("Never") - .endSpec() - .endTemplate() - .endSpec() - .build() - - return k8sClient - .batchV1Api() - .createNamespacedJob(namespace, body, null, null, null,null) - } - - /** - * Get a Jobs Job. - * - * @param name The job name - * @return An instance of {@link V1Job} - */ - @Override - V1Job getJob(String name) { - k8sClient - .batchV1Api() - .readNamespacedJob(name, namespace, null) - } - /** * Get a Job status * @@ -307,31 +250,7 @@ class K8sServiceImpl implements K8sService { .subPath(rel) } - /** - * Create a container for container image building via buildkit - * - * @param name - * The name of pod - * @param containerImage - * The container image to be used - * @param args - * The build command to be performed - * @param workDir - * The build context directory - * @param creds - * The target container repository credentials - * @return - * The {@link V1Pod} description the submitted pod - */ - @Override @Deprecated - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector) - return k8sClient - .coreV1Api() - .createNamespacedPod(namespace, spec, null, null, null,null) - } - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, Map nodeSelector) { // dirty dependency to avoid introducing another parameter @@ -408,47 +327,6 @@ class K8sServiceImpl implements K8sService { builder.build() } - /** - * Wait for a pod a completion. - * - * NOTE: this method assumes the pod is running exactly *one* container. - * - * @param pod - * The pod name - * @param timeout - * Max wait time in milliseconds - * @return - * An Integer value representing the container exit code or {@code null} if the state cannot be determined - * or timeout was reached. - */ - @Override - @Deprecated - Integer waitPodCompletion(V1Pod pod, long timeout) { - final start = System.currentTimeMillis() - // wait for termination - while( true ) { - final phase = pod.status?.phase - if( phase && phase != 'Pending' ) { - final status = pod.status.containerStatuses.first() - if( !status ) - return null - if( !status.state ) - return null - if( status.state.terminated ) { - return status.state.terminated.exitCode - } - } - - if( phase == 'Failed' ) - return null - final delta = System.currentTimeMillis()-start - if( delta > timeout ) - return null - sleep 5_000 - pod = getPod(pod.metadata.name) - } - } - /** * Fetch the logs of a pod. * @@ -481,36 +359,6 @@ class K8sServiceImpl implements K8sService { .deleteNamespacedPod(name, namespace, (String)null, (String)null, (Integer)null, (Boolean)null, (String)null, (V1DeleteOptions)null) } - /** - * Delete a pod where the status is reached - * - * @param name The name of the pod to be deleted - * @param statusName The status to be reached - * @param timeout The max wait time in milliseconds - */ - @Override - @Deprecated - void deletePodWhenReachStatus(String podName, String statusName, long timeout){ - final pod = getPod(podName) - final start = System.currentTimeMillis() - while( (System.currentTimeMillis() - start) < timeout ) { - if( pod?.status?.phase == statusName ) { - deletePod(podName) - return - } - sleep 5_000 - } - } - - @Override - @Deprecated - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { - final spec = scanSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) - return k8sClient - .coreV1Api() - .createNamespacedPod(namespace, spec, null, null, null,null) - } - @Deprecated V1Pod scanSpec(String name, String containerImage, List args, Path workDir, Path credsFile, ScanConfig scanConfig, Map nodeSelector) { @@ -865,33 +713,6 @@ class K8sServiceImpl implements K8sService { return result } - /** - * Wait for a job to complete - * - * @param k8s job - * @param timeout - * Max wait time in milliseconds - * @return list of pods created by the job - */ - @Deprecated - @Override - V1PodList waitJob(V1Job job, Long timeout) { - sleep 5_000 - final startTime = System.currentTimeMillis() - // wait for termination - while (System.currentTimeMillis() - startTime < timeout) { - final name = job.metadata.name - final status = getJobStatus(name) - if (status != JobStatus.Pending) { - return k8sClient - .coreV1Api() - .listNamespacedPod(namespace, null, null, null, null, "job-name=$name", null, null, null, null, null, null) - } - job = getJob(name) - } - return null - } - /** * Delete a job * diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy index 1dce14f26..83fab8470 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy @@ -25,7 +25,6 @@ import java.time.OffsetDateTime import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1PodStatus import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.service.blob.BlobEntry @@ -48,11 +47,8 @@ class KubeTransferStrategyTest extends Specification { def podName = "$jobName-abc".toString() def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) pod.status = new V1PodStatus(phase: "Succeeded") - def podList = new V1PodList(items: [pod]) k8sService.launchTransferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) - k8sService.waitJob(_, _) >> podList k8sService.getPod(_) >> pod - k8sService.waitPodCompletion(_, _) >> 0 k8sService.logsPod(_) >> "transfer successful" when: diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 4ebdd7c34..ddf92e6d9 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -36,13 +36,12 @@ import io.kubernetes.client.openapi.models.V1JobStatus import io.kubernetes.client.openapi.models.V1ObjectMeta import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList -import io.kubernetes.client.openapi.models.V1PodStatus import io.micronaut.context.ApplicationContext import io.micronaut.context.annotation.Replaces import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.MirrorConfig +import io.seqera.wave.configuration.ScanConfig /** * * @author Paolo Di Tommaso @@ -522,58 +521,6 @@ class K8sServiceImplTest extends Specification { ctx.close() } - def "deletePodWhenReachStatus should delete pod when status is reached within timeout"() { - given: - def podName = "test-pod" - def statusName = "Succeeded" - def timeout = 5000 - def api = Mock(CoreV1Api) - api.readNamespacedPod(_,_,_) >> new V1Pod(status: new V1PodStatus(phase: statusName)) - def k8sClient = new K8sClient() { - @Override - ApiClient apiClient() { - return null - } - CoreV1Api coreV1Api() { - return api - } - } - - def k8sService = new K8sServiceImpl(k8sClient: k8sClient) - - when: - k8sService.deletePodWhenReachStatus(podName, statusName, timeout) - - then: - 1 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) - } - - def "deletePodWhenReachStatus should not delete pod if status is not reached within timeout"() { - given: - def podName = "test-pod" - def statusName = "Succeeded" - def timeout = 5000 - def api = Mock(CoreV1Api) - api.readNamespacedPod(_,_,_) >> new V1Pod(status: new V1PodStatus(phase: "Running")) - def k8sClient = new K8sClient() { - @Override - ApiClient apiClient() { - return null - } - CoreV1Api coreV1Api() { - return api - } - } - - def k8sService = new K8sServiceImpl(k8sClient: k8sClient) - - when: - k8sService.deletePodWhenReachStatus(podName, statusName, timeout) - - then: - 0 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) - } - def "getLatestPodForJob should return the latest pod when multiple pods are present"() { given: def jobName = "test-job" From 2ee0854ecf75057fc07ea2ab48a426e8076009ff Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 6 Nov 2024 00:06:16 +0100 Subject: [PATCH 28/54] Bump jedis 5.1.3 (#732) Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 740119658..e70e5c513 100644 --- a/build.gradle +++ b/build.gradle @@ -60,7 +60,7 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation 'com.squareup.moshi:moshi:1.14.0' implementation 'com.squareup.moshi:moshi-adapters:1.14.0' - implementation 'redis.clients:jedis:5.0.2' + implementation 'redis.clients:jedis:5.1.3' implementation "io.github.resilience4j:resilience4j-ratelimiter:0.17.0" // caching deps implementation("io.micronaut.cache:micronaut-cache-core") From 2a342b18b0c41ad59ce5674d98863de8fd2d6323 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 6 Nov 2024 00:10:14 +0100 Subject: [PATCH 29/54] Bump make deps runtimeclasspath [ci skip] Signed-off-by: Paolo Di Tommaso --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 23895e870..ab044536b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -config ?= compileClasspath +config ?= runtimeClasspath ifdef module mm = :${module}: From f67e8556989402bd17606250ee3daaf07411ba0c Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Wed, 6 Nov 2024 04:53:22 +0530 Subject: [PATCH 30/54] Upgrade to Micronaut 4.6 (#318) Signed-off-by: munishchouhan Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- build.gradle | 21 +++++++----- ...wave.groovy-application-conventions.gradle | 4 +-- ...qera.wave.groovy-common-conventions.gradle | 6 ++-- ...eqera.wave.java-library-conventions.gradle | 30 ++++++++-------- gradle.properties | 2 +- .../auth/BasicAuthenticationProvider.groovy | 34 ++++++++----------- .../io/seqera/wave/auth/RegistryConfig.groovy | 2 +- .../wave/configuration/ScanConfig.groovy | 6 +++- .../controller/ContainerController.groovy | 5 +-- .../wave/controller/InspectController.groovy | 3 +- .../wave/controller/ValidateController.groovy | 6 ++-- .../ValidateRegistryCredsRequest.groovy | 4 +-- .../wave/exchange/PairingRequest.groovy | 5 ++- .../io/seqera/wave/proxy/ErrResponse.groovy | 6 ++-- .../impl/SpillWayStorageFactory.groovy | 4 +-- .../ratelimit/impl/SpillwayRateLimiter.groovy | 5 ++- .../service/account/AccountServiceImpl.groovy | 2 +- .../pairing/socket/PairingWebSocket.groovy | 3 ++ .../groovy/io/seqera/wave/tower/User.groovy | 6 ++-- .../controller/ContainerControllerTest.groovy | 1 - .../RegistryControllerLocalTest.groovy | 11 +++--- .../service/CredentialsServiceTest.groovy | 9 ++--- .../builder/KubeBuildStrategyTest.groovy | 2 -- .../service/k8s/K8sServiceImplTest.groovy | 11 ------ .../scan/DockerScanStrategyTest.groovy | 29 +++++++++------- 25 files changed, 104 insertions(+), 113 deletions(-) diff --git a/build.gradle b/build.gradle index e70e5c513..a02339f0a 100644 --- a/build.gradle +++ b/build.gradle @@ -2,10 +2,10 @@ import java.time.OffsetDateTime import java.time.format.DateTimeFormatter plugins { - id 'java-library' + id 'io.seqera.wave.java-library-conventions' id 'io.seqera.wave.groovy-application-conventions' - id "com.github.johnrengelman.shadow" version "7.1.1" - id "io.micronaut.minimal.application" version "3.7.0" + id "com.github.johnrengelman.shadow" version "8.1.1" + id "io.micronaut.minimal.application" version "4.1.1" id "com.google.cloud.tools.jib" version "3.4.2" id 'org.asciidoctor.jvm.convert' version '3.3.2' id 'jacoco' @@ -29,12 +29,13 @@ repositories { } dependencies { + annotationProcessor("io.micronaut.validation:micronaut-validation-processor") annotationProcessor("io.micronaut:micronaut-http-validation") compileOnly("io.micronaut.data:micronaut-data-processor") compileOnly("io.micronaut:micronaut-inject-groovy") compileOnly("io.micronaut:micronaut-http-validation") implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") - api 'io.seqera:lib-mail:1.0.0' + api 'io.seqera:lib-mail:1.1.0' api 'io.seqera:wave-api:0.13.3' api 'io.seqera:wave-utils:0.14.1' implementation("io.micronaut:micronaut-http-client") @@ -43,13 +44,14 @@ dependencies { implementation("io.micronaut.reactor:micronaut-reactor") implementation("io.micronaut.reactor:micronaut-reactor-http-client") implementation("jakarta.annotation:jakarta.annotation-api") - implementation("io.micronaut:micronaut-validation") + implementation("io.micronaut.validation:micronaut-validation") implementation 'io.micronaut.security:micronaut-security' - implementation "org.codehaus.groovy:groovy-json" - implementation "org.codehaus.groovy:groovy-nio" + implementation "org.apache.groovy:groovy-json" + implementation "org.apache.groovy:groovy-nio" implementation 'com.google.guava:guava:32.1.2-jre' implementation 'dev.failsafe:failsafe:3.1.0' - implementation('io.projectreactor:reactor-core') + implementation 'io.micronaut.reactor:micronaut-reactor' + implementation 'io.micronaut.reactor:micronaut-reactor-http-client' implementation("io.seqera:tower-crypto:22.4.0-watson") { transitive = false } // to be replaced with 22.4.0 once released implementation 'org.apache.commons:commons-compress:1.27.1' implementation 'org.apache.commons:commons-lang3:3.12.0' @@ -62,6 +64,7 @@ dependencies { implementation 'com.squareup.moshi:moshi-adapters:1.14.0' implementation 'redis.clients:jedis:5.1.3' implementation "io.github.resilience4j:resilience4j-ratelimiter:0.17.0" + implementation("io.micronaut:micronaut-retry") // caching deps implementation("io.micronaut.cache:micronaut-cache-core") implementation("io.micronaut.cache:micronaut-cache-caffeine") @@ -85,7 +88,7 @@ dependencies { testImplementation("org.testcontainers:mysql:1.17.3") // -- - implementation("ch.qos.logback:logback-classic:1.4.8") + implementation("ch.qos.logback:logback-classic:1.5.3") // rate limit implementation 'com.coveo:spillway:3.0.0' diff --git a/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle index 0cf39fc5e..c14579b32 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle @@ -12,7 +12,7 @@ plugins { group = 'io.seqera' -tasks.withType(Test) { +tasks.withType(Test).configureEach { // note: "--enable-preview" is required to use virtual thread on Java 19 and 20 - jvmArgs (["--enable-preview"]) + jvmArgs(["--enable-preview"]) } diff --git a/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle index da3059f91..915567a14 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle @@ -22,9 +22,9 @@ compileJava { options.release.set(11) } -tasks.withType(GroovyCompile) { - sourceCompatibility = '11' - targetCompatibility = '11' +tasks.withType(GroovyCompile).configureEach { + sourceCompatibility = '17' + targetCompatibility = '17' } group = 'io.seqera' diff --git a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle index f6197b641..8a690aede 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle @@ -24,9 +24,9 @@ compileJava { options.release.set(11) } -tasks.withType(GroovyCompile) { - sourceCompatibility = '11' - targetCompatibility = '11' +tasks.withType(GroovyCompile).configureEach { + sourceCompatibility = '17' + targetCompatibility = '17' } test { @@ -40,21 +40,21 @@ java { } dependencies { - implementation 'org.slf4j:slf4j-api:1.7.36' + implementation 'org.slf4j:slf4j-api:2.0.13' - testImplementation 'ch.qos.logback:logback-core:1.2.11' - testImplementation 'ch.qos.logback:logback-classic:1.2.11' - testImplementation "org.codehaus.groovy:groovy:3.0.15" - testImplementation "org.codehaus.groovy:groovy-nio:3.0.15" - testImplementation ("org.codehaus.groovy:groovy-test:3.0.17") - testImplementation ("cglib:cglib-nodep:3.3.0") - testImplementation ("org.objenesis:objenesis:3.2") - testImplementation ("org.spockframework:spock-core:2.3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('org.spockframework:spock-junit4:2.3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } + testImplementation 'ch.qos.logback:logback-core:1.4.11' + testImplementation 'ch.qos.logback:logback-classic:1.4.11' + testImplementation 'org.apache.groovy:groovy:4.0.15' + testImplementation 'org.apache.groovy:groovy-nio:4.0.15' + testImplementation 'org.apache.groovy:groovy-test:4.0.15' + testImplementation 'org.objenesis:objenesis:3.4' + testImplementation 'net.bytebuddy:byte-buddy:1.14.17' + testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0' + testImplementation 'org.spockframework:spock-junit4:2.3-groovy-4.0' } -tasks.withType(Test) { - jvmArgs ([ +tasks.withType(Test).configureEach { + jvmArgs([ '--enable-preview', '--add-opens=java.base/java.lang=ALL-UNNAMED', '--add-opens=java.base/java.io=ALL-UNNAMED', diff --git a/gradle.properties b/gradle.properties index 645585ab7..54b62b375 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,5 +16,5 @@ # along with this program. If not, see . # -micronautVersion=3.10.3 +micronautVersion=4.6.3 micronautEnvs=dev,h2,mail,aws-ses diff --git a/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy b/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy index b76abda4e..d76ff89ee 100644 --- a/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy +++ b/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy @@ -19,18 +19,17 @@ package io.seqera.wave.auth import groovy.util.logging.Slf4j +import io.micronaut.core.annotation.NonNull import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpRequest -import io.micronaut.security.authentication.AuthenticationProvider +import io.micronaut.security.authentication.AuthenticationFailureReason import io.micronaut.security.authentication.AuthenticationRequest import io.micronaut.security.authentication.AuthenticationResponse +import io.micronaut.security.authentication.provider.HttpRequestAuthenticationProvider import io.seqera.wave.service.account.AccountService import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Singleton -import org.reactivestreams.Publisher -import reactor.core.publisher.Flux -import reactor.core.publisher.FluxSink /** * Basic Authentication provider * @@ -38,25 +37,22 @@ import reactor.core.publisher.FluxSink */ @Slf4j @Singleton -class BasicAuthenticationProvider implements AuthenticationProvider { +class BasicAuthenticationProvider implements HttpRequestAuthenticationProvider { @Inject private AccountService accountService @Override - Publisher authenticate(@Nullable HttpRequest httpRequest, AuthenticationRequest authRequest) { - Flux.create(emitter -> { - final user = authRequest.identity?.toString() - final pass = authRequest.secret?.toString() - if (accountService.isAuthorised(user, pass)) { - log.trace "Auth request OK - user '$user'; password: '${StringUtils.redact(pass)}'" - emitter.next(AuthenticationResponse.success((String) authRequest.identity)) - emitter.complete() - } - else { - log.trace "Auth request FAILED - user '$user'; password: '${StringUtils.redact(pass)}'" - emitter.error(AuthenticationResponse.exception()) - } - }, FluxSink.OverflowStrategy.ERROR) + AuthenticationResponse authenticate(@Nullable HttpRequest httpRequest, @NonNull AuthenticationRequest authRequest) { + final user = authRequest.identity?.toString() + final pass = authRequest.secret?.toString() + if (accountService.isAuthorised(user, pass)) { + log.trace "Auth request OK - user '$user'; password: '${StringUtils.redact(pass)}'" + return AuthenticationResponse.success(authRequest.identity) + } + else { + log.trace "Auth request FAILED - user '$user'; password: '${StringUtils.redact(pass)}'" + return AuthenticationResponse.failure(AuthenticationFailureReason.CREDENTIALS_DO_NOT_MATCH) + } } } diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy index eb43f3dad..560a784a5 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy @@ -47,7 +47,7 @@ class RegistryConfig { * io: [ ... ] * ] */ - private Map registries + Map registries RegistryKeys getRegistryKeys(String registryName) { final String defaultRegistry = registries.get('default')?.toString() ?: 'docker.io' diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index baef40d75..5a5ac6006 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -95,7 +95,11 @@ class ScanConfig { @Memoized Path getCacheDirectory() { final result = Path.of(buildDirectory).toAbsolutePath().resolve('.trivy-cache') - Files.createDirectories(result) + try { + Files.createDirectories(result) + } catch (IOException e) { + log.error "Unable to create scan cache directory=${result} - cause: ${e.message}" + } return result } diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index fabef4017..10f71ddb4 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -29,6 +29,7 @@ import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpRequest import io.micronaut.http.HttpResponse +import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Delete import io.micronaut.http.annotation.Error @@ -181,13 +182,13 @@ class ContainerController { @Deprecated @Post('/container-token') @ExecuteOn(TaskExecutors.IO) - CompletableFuture> getToken(HttpRequest httpRequest, SubmitContainerTokenRequest req) { + CompletableFuture> getToken(HttpRequest httpRequest, @Body SubmitContainerTokenRequest req) { return getContainerImpl(httpRequest, req, false) } @Post('/v1alpha2/container') @ExecuteOn(TaskExecutors.IO) - CompletableFuture> getTokenV2(HttpRequest httpRequest, SubmitContainerTokenRequest req) { + CompletableFuture> getTokenV2(HttpRequest httpRequest, @Body SubmitContainerTokenRequest req) { return getContainerImpl(httpRequest, req, true) } diff --git a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy index 88d66f9b8..35e50bc58 100644 --- a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy @@ -25,6 +25,7 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpResponse +import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Post import io.micronaut.http.annotation.QueryValue @@ -70,7 +71,7 @@ class InspectController { private String serverUrl @Post("/v1alpha1/inspect") - CompletableFuture> inspect(ContainerInspectRequest req, @Nullable @QueryValue String platform) { + CompletableFuture> inspect(@Body ContainerInspectRequest req, @Nullable @QueryValue String platform) { if( !req.containerImage ) throw new BadRequestException("Missing 'containerImage' attribute") diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy index 631d0bcc8..f12e794dd 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy @@ -18,14 +18,14 @@ package io.seqera.wave.controller -import javax.validation.Valid - +import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Post import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.auth.RegistryAuthService import jakarta.inject.Inject +import jakarta.validation.Valid import reactor.core.publisher.Mono @ExecuteOn(TaskExecutors.IO) @@ -35,7 +35,7 @@ class ValidateController { @Inject RegistryAuthService loginService @Post - Mono validateCreds(@Valid ValidateRegistryCredsRequest request){ + Mono validateCreds(@Valid @Body ValidateRegistryCredsRequest request){ Mono.just( loginService.validateUser(request.registry, request.userName, request.password) ) diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy index db465ef0a..3fbe92d07 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy @@ -18,10 +18,8 @@ package io.seqera.wave.controller -import io.micronaut.core.annotation.Nullable -import javax.validation.constraints.NotBlank - import io.micronaut.core.annotation.Introspected +import jakarta.validation.constraints.NotBlank @Introspected class ValidateRegistryCredsRequest { diff --git a/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy b/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy index b54ac876d..fb17c68ec 100644 --- a/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy +++ b/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy @@ -18,11 +18,10 @@ package io.seqera.wave.exchange -import javax.validation.constraints.NotBlank -import javax.validation.constraints.NotNull - import groovy.transform.CompileStatic import io.micronaut.core.annotation.Introspected +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.NotNull /** * Model the request for a remote service instance to register diff --git a/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy b/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy index 7283fb205..5905f539e 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy @@ -90,17 +90,17 @@ class ErrResponse implements HttpResponse { } static ErrResponse forString(String msg, HttpRequest request) { - final head = HttpHeaders.of('Content-Type': ['text/plain'], {true}) + final head = HttpHeaders.of('Content-Type': ['text/plain'], (a, b) -> true) new ErrResponse(statusCode: 400, body: msg, request: request, uri: request.uri(), headers: head) } static ErrResponse forStream(String msg, HttpRequest request) { - final head = HttpHeaders.of('Content-Type': ['text/plain'], {true}) + final head = HttpHeaders.of('Content-Type': ['text/plain'], (a, b) -> true) new ErrResponse(statusCode: 400, body: new ByteArrayInputStream(msg.bytes), request: request, uri: request.uri(), headers: head) } static ErrResponse forByteArray(String msg, HttpRequest request) { - final head = HttpHeaders.of('Content-Type': ['text/plain'], {true}) + final head = HttpHeaders.of('Content-Type': ['text/plain'], (a, b) -> true) new ErrResponse(statusCode: 400, body: msg.bytes, request: request, uri: request.uri(), headers: head) } } diff --git a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy index c7431642f..cc6f9d036 100644 --- a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy +++ b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy @@ -18,8 +18,6 @@ package io.seqera.wave.ratelimit.impl -import javax.validation.constraints.NotNull - import com.coveo.spillway.storage.InMemoryStorage import com.coveo.spillway.storage.LimitUsageStorage import com.coveo.spillway.storage.RedisStorage @@ -27,9 +25,9 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Requires -import io.seqera.wave.configuration.RateLimiterConfig import io.seqera.wave.configuration.RedisConfig import jakarta.inject.Singleton +import jakarta.validation.constraints.NotNull import redis.clients.jedis.JedisPool /** diff --git a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy index 358e951dd..e2c9eab60 100644 --- a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy +++ b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy @@ -18,8 +18,6 @@ package io.seqera.wave.ratelimit.impl -import javax.validation.constraints.NotNull - import com.coveo.spillway.Spillway import com.coveo.spillway.SpillwayFactory import com.coveo.spillway.limit.Limit @@ -34,6 +32,7 @@ import io.seqera.wave.exception.SlowDownException import io.seqera.wave.ratelimit.AcquireRequest import io.seqera.wave.ratelimit.RateLimiterService import jakarta.inject.Singleton +import jakarta.validation.constraints.NotNull /** * This class manage how many requests can be requested from an user during a configurable period * @@ -60,7 +59,7 @@ class SpillwayRateLimiter implements RateLimiterService { init(storage, config) } - protected void init(@NotNull LimitUsageStorage storage, @NotNull RateLimiterConfig config){ + protected void init(LimitUsageStorage storage, RateLimiterConfig config){ SpillwayFactory spillwayFactory = new SpillwayFactory(storage) initBuilds(spillwayFactory, config) initPulls(spillwayFactory, config) diff --git a/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy index 3cccda9ab..f8d6bb4f1 100644 --- a/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy @@ -36,7 +36,7 @@ import jakarta.annotation.PostConstruct @ConfigurationProperties('wave') class AccountServiceImpl implements AccountService { - private Map accounts = Map.of() + Map accounts = Map.of() @PostConstruct private dumpAccounts() { diff --git a/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy b/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy index 1605e3b96..a4f7b0785 100644 --- a/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy +++ b/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy @@ -25,6 +25,8 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.micronaut.http.client.exceptions.HttpClientResponseException +import io.micronaut.scheduling.TaskExecutors +import io.micronaut.scheduling.annotation.ExecuteOn import io.micronaut.websocket.CloseReason import io.micronaut.websocket.WebSocketSession import io.micronaut.websocket.annotation.OnClose @@ -47,6 +49,7 @@ import static io.seqera.wave.util.LongRndKey.rndHex @Slf4j @CompileStatic @Singleton +@ExecuteOn(TaskExecutors.IO) @ServerWebSocket("/pairing/{service}/token/{token}{?endpoint}") class PairingWebSocket { diff --git a/src/main/groovy/io/seqera/wave/tower/User.groovy b/src/main/groovy/io/seqera/wave/tower/User.groovy index c8dbb71d8..bce993103 100644 --- a/src/main/groovy/io/seqera/wave/tower/User.groovy +++ b/src/main/groovy/io/seqera/wave/tower/User.groovy @@ -18,12 +18,12 @@ package io.seqera.wave.tower -import javax.validation.constraints.NotNull -import javax.validation.constraints.Size - import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import jakarta.validation.constraints.NotNull +import jakarta.validation.constraints.Size + /** * Model a tower user * diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 1a4658f62..d582316b8 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -72,7 +72,6 @@ import jakarta.inject.Inject * @author Paolo Di Tommaso */ @MicronautTest -@Property(name='wave.build.workspace', value='/some/wsp') @Property(name='wave.build.repo', value='wave/build') @Property(name='wave.build.cache', value='wave/build/cache') class ContainerControllerTest extends Specification { diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy index ef37f10ba..9057ac53e 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy @@ -142,16 +142,19 @@ class RegistryControllerLocalTest extends Specification implements DockerRegistr h.add('Accept', it) } }) - HttpResponse response = client.toBlocking().exchange(request, Map) + HttpResponse response = client.toBlocking().exchange(request, String) then: response.status() == HttpStatus.OK when: - def list = response.body().manifests.collect{ - String type = it.mediaType.indexOf("manifest") ? "manifests" : "blobs" - "/v2/$IMAGE/$type/$it.digest" + def parsedBody = new JsonSlurper().parseText(response.body.get()) as Map + and: + def list = parsedBody.manifests.collect { + String type = it.mediaType.contains("manifest") ? "manifests" : "blobs" + return "/v2/$IMAGE/$type/$it.digest" as String } + and: boolean fails = list.find{ url -> HttpRequest requestGet = HttpRequest.GET(url).headers({ h -> accept.each { diff --git a/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy index a2babb87d..66b2012b2 100644 --- a/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy @@ -51,11 +51,9 @@ class CredentialsServiceTest extends Specification { @Inject CredentialsService credentialsService - @MockBean(TowerClient) TowerClient towerClient = Mock(TowerClient) - @MockBean(PairingService) PairingService securityService = Mock(PairingService) @@ -121,7 +119,6 @@ class CredentialsServiceTest extends Specification { noExceptionThrown() } - def 'should fail if keys where not registered for the tower endpoint'() { given: def identity = new PlatformId(new User(id:10), 10,"token",'endpoint') @@ -169,7 +166,7 @@ class CredentialsServiceTest extends Specification { registry: 'docker.io' ) and: - def identity = new PlatformId(new User(id:10), 10,"token",'tower.io', '101') + def identity = new PlatformId(new User(id:10), 100, 'token', 'tower.io', '101') def auth = JwtAuth.of(identity) when: @@ -185,7 +182,7 @@ class CredentialsServiceTest extends Specification { ) and: 'non matching credentials are listed' - 1 * towerClient.listCredentials('tower.io',auth,10) >> CompletableFuture.completedFuture(new ListCredentialsResponse( + 1 * towerClient.listCredentials('tower.io',auth,100) >> CompletableFuture.completedFuture(new ListCredentialsResponse( credentials: [nonContainerRegistryCredentials,otherRegistryCredentials] )) @@ -223,7 +220,7 @@ class CredentialsServiceTest extends Specification { def 'should get registry creds from compute creds when not found in tower credentials'() { given: 'a tower user in a workspace on a specific instance with a valid token' def userId = 10 - def workspaceId = 10 + def workspaceId = 100 def token = "valid-token" def towerEndpoint = "http://tower.io:9090" def workflowId = "id123" diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 74ffaae38..2c9e221e2 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -38,11 +38,9 @@ import jakarta.inject.Inject * @author Paolo Di Tommaso */ @MicronautTest -@Property(name="wave.build.workspace",value="/build/work") @Property(name="wave.build.k8s.namespace",value="foo") @Property(name="wave.build.k8s.configPath",value="/home/kube.config") @Property(name="wave.build.k8s.storage.claimName",value="bar") -@Property(name="wave.build.k8s.storage.mountPath",value="/build") @Property(name='wave.build.k8s.node-selector[linux/amd64]',value="service=wave-build") @Property(name='wave.build.k8s.node-selector[linux/arm64]',value="service=wave-build-arm64") class KubeBuildStrategyTest extends Specification { diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index ddf92e6d9..d4cb2e061 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -37,8 +37,6 @@ import io.kubernetes.client.openapi.models.V1ObjectMeta import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList import io.micronaut.context.ApplicationContext -import io.micronaut.context.annotation.Replaces -import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.configuration.ScanConfig @@ -46,17 +44,8 @@ import io.seqera.wave.configuration.ScanConfig * * @author Paolo Di Tommaso */ -@MicronautTest class K8sServiceImplTest extends Specification { - @Replaces(ScanConfig.class) - static class MockScanConfig extends ScanConfig { - @Override - Path getCacheDirectory() { - return Path.of('/build/scan/cache') - } - } - def 'should validate context OK ' () { when: def PROPS = [ diff --git a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy index 5e9c898e5..1a759cc58 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy @@ -20,24 +20,30 @@ package io.seqera.wave.service.scan import spock.lang.Specification -import java.nio.file.Files import java.nio.file.Path -import io.micronaut.context.ApplicationContext +import io.micronaut.test.annotation.MockBean +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.ScanConfig +import jakarta.inject.Inject /** * * @author Munish Chouhan */ +@MicronautTest class DockerScanStrategyTest extends Specification { + @Inject + DockerScanStrategy dockerContainerStrategy + + @MockBean(ScanConfig) + ScanConfig mockConfig() { + Mock(ScanConfig) { + getCacheDirectory() >> Path.of('/some/scan/cache') + } + } + def 'should get docker command' () { - given: - def workspace = Files.createTempDirectory('test') - def props = ['wave.build.workspace': workspace.toString()] - and: - def ctx = ApplicationContext.run(props) - and: - def dockerContainerStrategy = ctx.getBean(DockerScanStrategy) when: def scanDir = Path.of('/some/scan/dir') @@ -56,7 +62,7 @@ class DockerScanStrategyTest extends Specification { '-v', '/some/scan/dir:/some/scan/dir:rw', '-v', - "/build/scan/cache:/root/.cache/:rw", + '/some/scan/cache:/root/.cache/:rw', '-v', '/user/test/build-workspace/config.json:/root/.docker/config.json:ro', '-e', @@ -65,8 +71,5 @@ class DockerScanStrategyTest extends Specification { 'BAR=2' ] - cleanup: - ctx.close() - workspace?.deleteDir() } } From 6aeb3c330d51ead83e2a4c257d57a299f68645d5 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 6 Nov 2024 00:36:08 +0100 Subject: [PATCH 31/54] Bump snakeyaml 2.2 Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a02339f0a..703892c45 100644 --- a/build.gradle +++ b/build.gradle @@ -72,7 +72,7 @@ dependencies { implementation "software.amazon.awssdk:ecr" implementation "software.amazon.awssdk:ecrpublic" implementation 'software.amazon.awssdk:ses' - implementation 'org.yaml:snakeyaml:2.0' + implementation 'org.yaml:snakeyaml:2.2' implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' implementation 'org.luaj:luaj-jse:3.0.1' //object storage dependency From f5fe3fa4837e98ea0a9f7c37b7c52b04ea539cd4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 6 Nov 2024 00:36:36 +0100 Subject: [PATCH 32/54] Bump logback 1.5.12 Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- .../groovy/io.seqera.wave.java-library-conventions.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 703892c45..d88578d69 100644 --- a/build.gradle +++ b/build.gradle @@ -88,7 +88,7 @@ dependencies { testImplementation("org.testcontainers:mysql:1.17.3") // -- - implementation("ch.qos.logback:logback-classic:1.5.3") + implementation("ch.qos.logback:logback-classic:1.5.12") // rate limit implementation 'com.coveo:spillway:3.0.0' diff --git a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle index 8a690aede..f6101c48e 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle @@ -42,8 +42,8 @@ java { dependencies { implementation 'org.slf4j:slf4j-api:2.0.13' - testImplementation 'ch.qos.logback:logback-core:1.4.11' - testImplementation 'ch.qos.logback:logback-classic:1.4.11' + testImplementation 'ch.qos.logback:logback-core:1.5.12' + testImplementation 'ch.qos.logback:logback-classic:1.5.12' testImplementation 'org.apache.groovy:groovy:4.0.15' testImplementation 'org.apache.groovy:groovy-nio:4.0.15' testImplementation 'org.apache.groovy:groovy-test:4.0.15' From ede22ce5ee65f34f4b832475b0c9d53b2c8e1fb2 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 6 Nov 2024 00:44:10 +0100 Subject: [PATCH 33/54] Bump bouncycastle:bcpkix-jdk18on:1.78 Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d88578d69..575d72186 100644 --- a/build.gradle +++ b/build.gradle @@ -81,7 +81,7 @@ dependencies { // this sts dependency is require by micronaut-aws-parameter-store, // not directly used by the app, for this reason keeping `runtimeOnly` runtimeOnly "software.amazon.awssdk:sts" - + runtimeOnly 'org.bouncycastle:bcpkix-jdk18on:1.78' runtimeOnly("io.netty:netty-tcnative-boringssl-static:2.0.0.Final") runtimeOnly("javax.xml.bind:jaxb-api:2.3.1") testImplementation("org.testcontainers:testcontainers") From 2e10416a84c1a97716455ccf9b73945466eea28e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 6 Nov 2024 00:48:49 +0100 Subject: [PATCH 34/54] Bump bitbucket.b_c:jose4j:0.9.4 Signed-off-by: Paolo Di Tommaso --- build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 575d72186..2043bc8ea 100644 --- a/build.gradle +++ b/build.gradle @@ -81,7 +81,6 @@ dependencies { // this sts dependency is require by micronaut-aws-parameter-store, // not directly used by the app, for this reason keeping `runtimeOnly` runtimeOnly "software.amazon.awssdk:sts" - runtimeOnly 'org.bouncycastle:bcpkix-jdk18on:1.78' runtimeOnly("io.netty:netty-tcnative-boringssl-static:2.0.0.Final") runtimeOnly("javax.xml.bind:jaxb-api:2.3.1") testImplementation("org.testcontainers:testcontainers") @@ -99,6 +98,10 @@ dependencies { implementation "io.micronaut:micronaut-management" //views implementation("io.micronaut.views:micronaut-views-handlebars") + + // upgrade indirect dependencies + runtimeOnly 'org.bouncycastle:bcpkix-jdk18on:1.78' + runtimeOnly 'org.bitbucket.b_c:jose4j:0.9.4' } application { From 2e413ac2ff8ef265168b85b2f9930a0f1c6984e7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 9 Nov 2024 12:49:38 +0100 Subject: [PATCH 35/54] Add explicit dep to websocket module Signed-off-by: Paolo Di Tommaso --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 2043bc8ea..a109e20ff 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,7 @@ dependencies { implementation("jakarta.annotation:jakarta.annotation-api") implementation("io.micronaut.validation:micronaut-validation") implementation 'io.micronaut.security:micronaut-security' + implementation 'io.micronaut:micronaut-websocket' implementation "org.apache.groovy:groovy-json" implementation "org.apache.groovy:groovy-nio" implementation 'com.google.guava:guava:32.1.2-jre' From 2450cef6eacbfb9bb90be58d1b7c9977cf59fbd6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 9 Nov 2024 12:53:02 +0100 Subject: [PATCH 36/54] Reformatting deps Signed-off-by: Paolo Di Tommaso --- build.gradle | 68 ++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/build.gradle b/build.gradle index a109e20ff..a6f8550b2 100644 --- a/build.gradle +++ b/build.gradle @@ -29,76 +29,76 @@ repositories { } dependencies { - annotationProcessor("io.micronaut.validation:micronaut-validation-processor") - annotationProcessor("io.micronaut:micronaut-http-validation") - compileOnly("io.micronaut.data:micronaut-data-processor") - compileOnly("io.micronaut:micronaut-inject-groovy") - compileOnly("io.micronaut:micronaut-http-validation") - implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") + annotationProcessor 'io.micronaut.validation:micronaut-validation-processor' + annotationProcessor 'io.micronaut:micronaut-http-validation' + compileOnly 'io.micronaut.data:micronaut-data-processor' + compileOnly 'io.micronaut:micronaut-inject-groovy' + compileOnly 'io.micronaut:micronaut-http-validation' + implementation 'jakarta.persistence:jakarta.persistence-api:3.0.0' api 'io.seqera:lib-mail:1.1.0' api 'io.seqera:wave-api:0.13.3' api 'io.seqera:wave-utils:0.14.1' - implementation("io.micronaut:micronaut-http-client") - implementation("io.micronaut:micronaut-jackson-databind") - implementation("io.micronaut.groovy:micronaut-runtime-groovy") - implementation("io.micronaut.reactor:micronaut-reactor") - implementation("io.micronaut.reactor:micronaut-reactor-http-client") - implementation("jakarta.annotation:jakarta.annotation-api") - implementation("io.micronaut.validation:micronaut-validation") + implementation 'io.micronaut:micronaut-http-client' + implementation 'io.micronaut:micronaut-jackson-databind' + implementation 'io.micronaut.groovy:micronaut-runtime-groovy' + implementation 'io.micronaut.reactor:micronaut-reactor' + implementation 'io.micronaut.reactor:micronaut-reactor-http-client' + implementation 'jakarta.annotation:jakarta.annotation-api' + implementation 'io.micronaut.validation:micronaut-validation' implementation 'io.micronaut.security:micronaut-security' implementation 'io.micronaut:micronaut-websocket' - implementation "org.apache.groovy:groovy-json" - implementation "org.apache.groovy:groovy-nio" + implementation 'org.apache.groovy:groovy-json' + implementation 'org.apache.groovy:groovy-nio' implementation 'com.google.guava:guava:32.1.2-jre' implementation 'dev.failsafe:failsafe:3.1.0' implementation 'io.micronaut.reactor:micronaut-reactor' implementation 'io.micronaut.reactor:micronaut-reactor-http-client' - implementation("io.seqera:tower-crypto:22.4.0-watson") { transitive = false } // to be replaced with 22.4.0 once released + implementation('io.seqera:tower-crypto:22.4.0-watson') { transitive = false } // to be replaced with 22.4.0 once released implementation 'org.apache.commons:commons-compress:1.27.1' implementation 'org.apache.commons:commons-lang3:3.12.0' implementation 'io.kubernetes:client-java:19.0.0' implementation 'io.kubernetes:client-java-api-fluent:18.0.1' implementation 'com.google.code.gson:gson:2.9.0' - implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation 'com.squareup.moshi:moshi:1.14.0' implementation 'com.squareup.moshi:moshi-adapters:1.14.0' implementation 'redis.clients:jedis:5.1.3' - implementation "io.github.resilience4j:resilience4j-ratelimiter:0.17.0" - implementation("io.micronaut:micronaut-retry") + implementation 'io.github.resilience4j:resilience4j-ratelimiter:0.17.0' + implementation 'io.micronaut:micronaut-retry' // caching deps - implementation("io.micronaut.cache:micronaut-cache-core") - implementation("io.micronaut.cache:micronaut-cache-caffeine") - implementation("io.micronaut.aws:micronaut-aws-parameter-store") - implementation "software.amazon.awssdk:ecr" - implementation "software.amazon.awssdk:ecrpublic" + implementation 'io.micronaut.cache:micronaut-cache-core' + implementation 'io.micronaut.cache:micronaut-cache-caffeine' + implementation 'io.micronaut.aws:micronaut-aws-parameter-store' + implementation 'software.amazon.awssdk:ecr' + implementation 'software.amazon.awssdk:ecrpublic' implementation 'software.amazon.awssdk:ses' implementation 'org.yaml:snakeyaml:2.2' implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' implementation 'org.luaj:luaj-jse:3.0.1' //object storage dependency - implementation("io.micronaut.objectstorage:micronaut-object-storage-aws") + implementation 'io.micronaut.objectstorage:micronaut-object-storage-aws' // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570 // this sts dependency is require by micronaut-aws-parameter-store, // not directly used by the app, for this reason keeping `runtimeOnly` - runtimeOnly "software.amazon.awssdk:sts" - runtimeOnly("io.netty:netty-tcnative-boringssl-static:2.0.0.Final") - runtimeOnly("javax.xml.bind:jaxb-api:2.3.1") - testImplementation("org.testcontainers:testcontainers") - testImplementation("org.testcontainers:mysql:1.17.3") + runtimeOnly 'software.amazon.awssdk:sts' + runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.0.Final' + runtimeOnly 'javax.xml.bind:jaxb-api:2.3.1' + testImplementation 'org.testcontainers:testcontainers' + testImplementation 'org.testcontainers:mysql:1.17.3' // -- - implementation("ch.qos.logback:logback-classic:1.5.12") + implementation 'ch.qos.logback:logback-classic:1.5.12' // rate limit implementation 'com.coveo:spillway:3.0.0' // monitoring - implementation "io.micronaut.micrometer:micronaut-micrometer-registry-prometheus" + implementation 'io.micronaut.micrometer:micronaut-micrometer-registry-prometheus' // Also required to enable endpoint - implementation "io.micronaut:micronaut-management" + implementation 'io.micronaut:micronaut-management' //views - implementation("io.micronaut.views:micronaut-views-handlebars") + implementation 'io.micronaut.views:micronaut-views-handlebars' // upgrade indirect dependencies runtimeOnly 'org.bouncycastle:bcpkix-jdk18on:1.78' From 132f94918a1c9592a26d2748f799f2a3aa560aea Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 9 Nov 2024 15:07:52 +0100 Subject: [PATCH 37/54] Bump Java 21 as build requirement (#519) Signed-off-by: Paolo Di Tommaso Co-authored-by: Munish Chouhan --- .github/workflows/build.yml | 2 +- README.md | 4 ++-- .../io.seqera.wave.groovy-application-conventions.gradle | 5 ----- .../groovy/io.seqera.wave.groovy-common-conventions.gradle | 4 ++-- .../groovy/io.seqera.wave.java-library-conventions.gradle | 5 ++--- settings.gradle | 7 +++++++ .../wave/service/validation/ValidationServiceImpl.groovy | 2 +- .../service/validation/ValidationServiceProdTest.groovy | 2 +- .../wave/service/validation/ValidationServiceTest.groovy | 2 +- 9 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 993808b58..4cf422e24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - java_version: [19] + java_version: [21] steps: - name: Environment diff --git a/README.md b/README.md index 10d853924..13acb4415 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ images. * Push and cache built containers to a user-provided container repository; * Build Singularity native containers both using a Singularity spec file, Conda package(s); * Push Singularity native container images to OCI-compliant registries; - +* Scan container images for security vulnerabilities ### How it works @@ -34,7 +34,7 @@ container registry where the image is stored, while the instrumented layers are ### Requirements -* Java 19 or later +* Java 21 or later * Linux or macOS * Redis 6.2 (or later) * Docker engine (for development) diff --git a/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle index c14579b32..4ff807f4c 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle @@ -11,8 +11,3 @@ plugins { } group = 'io.seqera' - -tasks.withType(Test).configureEach { - // note: "--enable-preview" is required to use virtual thread on Java 19 and 20 - jvmArgs(["--enable-preview"]) -} diff --git a/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle index 915567a14..45696fae9 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle @@ -14,12 +14,12 @@ repositories { java { toolchain { - languageVersion = JavaLanguageVersion.of(19) + languageVersion = JavaLanguageVersion.of(21) } } compileJava { - options.release.set(11) + options.release.set(17) } tasks.withType(GroovyCompile).configureEach { diff --git a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle index f6101c48e..6763c4926 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle @@ -16,12 +16,12 @@ repositories { java { toolchain { - languageVersion = JavaLanguageVersion.of(19) + languageVersion = JavaLanguageVersion.of(21) } } compileJava { - options.release.set(11) + options.release.set(17) } tasks.withType(GroovyCompile).configureEach { @@ -55,7 +55,6 @@ dependencies { tasks.withType(Test).configureEach { jvmArgs([ - '--enable-preview', '--add-opens=java.base/java.lang=ALL-UNNAMED', '--add-opens=java.base/java.io=ALL-UNNAMED', '--add-opens=java.base/java.nio=ALL-UNNAMED', diff --git a/settings.gradle b/settings.gradle index 395412606..0605b3681 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,10 @@ +plugins { + // required to download the toolchain (jdk) from a remote repository + // https://github.com/gradle/foojay-toolchains + // https://docs.gradle.org/current/userguide/toolchains.html#sub:download_repositories + id("org.gradle.toolchains.foojay-resolver-convention") version "0.7.0" +} + rootProject.name="wave" // only for development diff --git a/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy index 0bc3a0c80..e87a51e4c 100644 --- a/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy @@ -55,7 +55,7 @@ class ValidationServiceImpl implements ValidationService { new URI(endpoint) } catch (URISyntaxException e) { - return "Invalid endpoint '${endpoint}' — cause: ${e.message}" + return "Invalid endpoint '${endpoint}' — cause: ${e.getMessage()}" } return null diff --git a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy index d0316fb0e..cc0174184 100644 --- a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy @@ -73,7 +73,7 @@ class ValidationServiceProdTest extends Specification { ENDPOINT | EXPECTED 'foo' | "Missing endpoint protocol — offending value: foo" 'ftp://foo.com' | "Invalid endpoint protocol — offending value: ftp://foo.com" - 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 7: http://a b c" + 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 8: http://a b c" 'http://localhost' | 'Endpoint hostname not allowed — offending value: http://localhost' 'http://localhost:8000' | 'Endpoint hostname not allowed — offending value: http://localhost:8000' 'http://10.0.0.0/api' | 'Endpoint hostname not allowed — offending value: http://10.0.0.0/api' diff --git a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy index 12ba6c1e1..4ed710744 100644 --- a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy @@ -54,7 +54,7 @@ class ValidationServiceTest extends Specification { ENDPOINT | EXPECTED 'foo' | "Missing endpoint protocol — offending value: foo" 'ftp://foo.com' | "Invalid endpoint protocol — offending value: ftp://foo.com" - 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 7: http://a b c" + 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 8: http://a b c" and: 'http://foo.com' | null 'http://localhost' | null From a2a9c4ea747647c89de126656544c37eb1545983 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 10 Nov 2024 13:54:05 +0100 Subject: [PATCH 38/54] Improve logging Signed-off-by: Paolo Di Tommaso --- src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy | 2 +- src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index 5a5ac6006..0ff20d39c 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -133,7 +133,7 @@ class ScanConfig { final name = p!=-1 ? entry.substring(0,p) : entry final value = p!=-1 ? entry.substring(p+1) : '' if( !value ) - log.warn "Invalid 'wave.scan.environment value' -- offending entry: '$entry'" + log.warn "Invalid 'wave.scan.environment' value -- offending entry: '$entry'" result.add(new Tuple2(name,value)) } return result diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index 9b6a9291e..33cad3b12 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -23,6 +23,7 @@ import java.time.Instant import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.ToString import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId @@ -32,6 +33,7 @@ import io.seqera.wave.tower.PlatformId * @author Paolo Di Tommaso */ @Canonical +@ToString(includeNames = true, includePackage = false) @CompileStatic class ScanRequest { From 76f0a456a792f2a5a9888ce60e1f0075e73337ce Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 10 Nov 2024 13:54:39 +0100 Subject: [PATCH 39/54] Fix K8s env propagation Signed-off-by: Paolo Di Tommaso --- .../groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 2 +- .../io/seqera/wave/service/k8s/K8sServiceImplTest.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index bd88f3974..c954794c2 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -636,7 +636,7 @@ class K8sServiceImpl implements K8sService { for( Tuple2 entry : env ) { final String k = entry.v1 final String v = entry.v2 - container.withEnv(new V1EnvVar().name(k).value(v)) + container.addToEnv(new V1EnvVar().name(k).value(v)) } // spec section diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index d4cb2e061..f1b276c47 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -684,7 +684,7 @@ class K8sServiceImplTest extends Specification { getCacheDirectory() >> Path.of('/build/cache/dir') getRequestsCpu() >> '2' getRequestsMemory() >> '4Gi' - getEnvironmentAsTuples() >> [new Tuple2('GITHUB_TOKEN', '123abc')] + getEnvironmentAsTuples() >> [new Tuple2('FOO', 'abc'), new Tuple2('BAR', 'xyz')] } when: @@ -697,7 +697,7 @@ class K8sServiceImplTest extends Specification { job.spec.template.spec.containers[0].args == args job.spec.template.spec.containers[0].resources.requests.get('cpu') == new Quantity('2') job.spec.template.spec.containers[0].resources.requests.get('memory') == new Quantity('4Gi') - job.spec.template.spec.containers[0].env == [ new V1EnvVar().name('GITHUB_TOKEN').value('123abc') ] + job.spec.template.spec.containers[0].env == [ new V1EnvVar().name('FOO').value('abc'), new V1EnvVar().name('BAR').value('xyz') ] job.spec.template.spec.volumes.size() == 1 job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' job.spec.template.spec.restartPolicy == 'Never' From 55b28f8b9d0e3fa8930a39707594010846f1f6fe Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 10 Nov 2024 16:48:44 +0100 Subject: [PATCH 40/54] [release] bump version 1.14.0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 66fae2a63..850e74240 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.11 +1.14.0 diff --git a/changelog.txt b/changelog.txt index 3085abccb..297b10e89 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,18 @@ # Wave changelog +1.14.0 - 10 Nov 2024 +- Fix K8s env propagation [76f0a456] +- Remove deprecated K8s methods (#734) [481298bf] +- Bump to Micronaut 4.6 (#318) [f67e8556] +- Bump Java 21 as build requirement (#519) [132f9491] +- Bump bitbucket.b_c:jose4j:0.9.4 [2e10416a] +- Bump bouncycastle:bcpkix-jdk18on:1.78 [ede22ce5] +- Bump jedis 5.1.3 (#732) [2ee0854e] +- Bump logback 1.5.12 [f5fe3fa4] +- Bump make deps runtimeclasspath [2a342b18] +- Bump snakeyaml 2.2 [6aeb3c33] +- Bump spillway 3.0.0 (#731) [1502696d] +- Bump explicit dep to websocket module [2e413ac2] + 1.13.11 - 2 Nov 2024 - Rename async methods for semantic consistency [38114d75] - Save scan record async (#730) [3ad82a3a] From a120db440a06db97a9801a59ee940f265a1667f2 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 11 Nov 2024 08:28:33 +0100 Subject: [PATCH 41/54] Update changelog Signed-off-by: Paolo Di Tommaso --- changelog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.txt b/changelog.txt index 297b10e89..a00e63639 100644 --- a/changelog.txt +++ b/changelog.txt @@ -12,6 +12,7 @@ - Bump snakeyaml 2.2 [6aeb3c33] - Bump spillway 3.0.0 (#731) [1502696d] - Bump explicit dep to websocket module [2e413ac2] +- Enables EKS Pod identity via AWS SDK 2.27.8 1.13.11 - 2 Nov 2024 - Rename async methods for semantic consistency [38114d75] From 5d594a24e038c559ca7c2502755bca8af884ae3f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 14 Nov 2024 17:07:00 +0100 Subject: [PATCH 42/54] Fix failint tests Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/core/RegistryProxyServiceTest.groovy | 4 ++-- .../groovy/io/seqera/wave/proxy/ProxyClientTest.groovy | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy b/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy index 44d2db027..68674e2fb 100644 --- a/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy @@ -57,7 +57,7 @@ class RegistryProxyServiceTest extends Specification { @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) def 'should retrieve image digest on ECR' () { given: - def IMAGE = '195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/kaniko:0.1.0' + def IMAGE = '195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/app:1.0.0' def request = Mock(BuildRequest) when: @@ -66,6 +66,6 @@ class RegistryProxyServiceTest extends Specification { request.getTargetImage() >> IMAGE request.getIdentity() >> new PlatformId() then: - resp1 == 'sha256:05f9dc67e6ec879773de726b800d4d5044f8bd8e67da728484fbdea56af1fdff' + resp1 == 'sha256:d5c169e210d6b789b2dc7eced66471cf4ce2c7260ac7299fbef464ff902086be' } } diff --git a/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy b/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy index a3e3328e3..f952cf4e8 100644 --- a/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy +++ b/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy @@ -145,7 +145,7 @@ class ProxyClientTest extends Specification { @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) def 'should call target manifest on amazon' () { given: - def IMAGE = 'wave/kaniko' + def IMAGE = 'wave/app' def REG = '195996028523.dkr.ecr.eu-west-1.amazonaws.com' def registry = lookupService.lookup(REG) def creds = credentialsProvider.getDefaultCredentials(REG) @@ -158,7 +158,7 @@ class ProxyClientTest extends Specification { .withCredentials(creds) when: - def resp = proxy.getString("/v2/$IMAGE/manifests/0.1.0") + def resp = proxy.getString("/v2/$IMAGE/manifests/1.0.0") then: resp.statusCode() == 200 } @@ -166,7 +166,7 @@ class ProxyClientTest extends Specification { @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) def 'should call head manifest on amazon' () { given: - def IMAGE = 'wave/kaniko' + def IMAGE = 'wave/app' def REG = '195996028523.dkr.ecr.eu-west-1.amazonaws.com' def registry = lookupService.lookup(REG) def creds = credentialsProvider.getDefaultCredentials(REG) @@ -179,7 +179,7 @@ class ProxyClientTest extends Specification { .withCredentials(creds) when: - def resp = proxy.head("/v2/$IMAGE/manifests/0.1.0") + def resp = proxy.head("/v2/$IMAGE/manifests/1.0.0") then: resp.statusCode() == 200 } From 8c0f3a4c39b73cd697a993fe3ac836f3d2d322aa Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 14 Nov 2024 17:59:44 +0100 Subject: [PATCH 43/54] Fix creds validation endpoint (#740) [ci skip] Signed-off-by: Paolo Di Tommaso --- .../wave/controller/ValidateController.groovy | 8 ++------ .../ValidateCredsControllerTest.groovy | 16 ++++++++-------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy index f12e794dd..f7136f572 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.controller -import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Post import io.micronaut.scheduling.TaskExecutors @@ -26,7 +25,6 @@ import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.auth.RegistryAuthService import jakarta.inject.Inject import jakarta.validation.Valid -import reactor.core.publisher.Mono @ExecuteOn(TaskExecutors.IO) @Controller("/validate-creds") @@ -35,10 +33,8 @@ class ValidateController { @Inject RegistryAuthService loginService @Post - Mono validateCreds(@Valid @Body ValidateRegistryCredsRequest request){ - Mono.just( - loginService.validateUser(request.registry, request.userName, request.password) - ) + Boolean validateCreds(@Valid ValidateRegistryCredsRequest request){ + loginService.validateUser(request.registry, request.userName, request.password) } } diff --git a/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy index 5f3cad31f..586d48616 100644 --- a/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy @@ -67,9 +67,9 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'should validate username required'() { when: - HttpRequest request = HttpRequest.POST("/validate-creds", [ + HttpRequest request = HttpRequest.POST("/validate-creds", [request: [ password: 'test', - ]) + ]]) client.toBlocking().exchange(request, Boolean) then: def e = thrown(HttpClientResponseException) @@ -77,9 +77,9 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'should validate pwd required'() { when: - HttpRequest request = HttpRequest.POST("/validate-creds", [ + HttpRequest request = HttpRequest.POST("/validate-creds", [request:[ userName: 'test', - ]) + ]]) client.toBlocking().exchange(request, Boolean) then: def e = thrown(HttpClientResponseException) @@ -87,10 +87,10 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'should validate the test user'() { given: - def req = [ + def req = [request: [ userName:'test', password:'test', - registry: getTestRegistryUrl('test') ] + registry: getTestRegistryUrl('test') ]] and: HttpRequest request = HttpRequest.POST("/validate-creds", req) when: @@ -103,11 +103,11 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'test validateController valid login'() { given: - def req = [ + def req = [request: [ userName: USER, password: PWD, registry: getTestRegistryUrl(REGISTRY_URL) - ] + ]] HttpRequest request = HttpRequest.POST("/validate-creds", req) when: HttpResponse response = client.toBlocking().exchange(request, Boolean) From 97f8a725bd0880703f689cc266c3248a4ae86ea3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 14 Nov 2024 18:01:09 +0100 Subject: [PATCH 44/54] [release] bump 1.14.1 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 850e74240..63e799cf4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.0 +1.14.1 diff --git a/changelog.txt b/changelog.txt index a00e63639..7190df430 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,7 @@ # Wave changelog +1.14.1 - 14 Nov 2024 +- Fix creds validation endpoint (#740) [8c0f3a4c] + 1.14.0 - 10 Nov 2024 - Fix K8s env propagation [76f0a456] - Remove deprecated K8s methods (#734) [481298bf] From 52272fe1169c3fc2ea87c895e7f61b3954e996c6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 16 Nov 2024 16:51:25 +0100 Subject: [PATCH 45/54] Bump gradle 8.10.2 Signed-off-by: Paolo Di Tommaso --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4413138c..df97d72b8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From f24b684d6e1f41e86793e8b998144f99eedd9df5 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 16 Nov 2024 18:04:41 +0100 Subject: [PATCH 46/54] Update project deps Signed-off-by: Paolo Di Tommaso --- build.gradle | 12 ++++++------ .../io.seqera.wave.java-library-conventions.gradle | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index a6f8550b2..ccbc223b5 100644 --- a/build.gradle +++ b/build.gradle @@ -35,9 +35,9 @@ dependencies { compileOnly 'io.micronaut:micronaut-inject-groovy' compileOnly 'io.micronaut:micronaut-http-validation' implementation 'jakarta.persistence:jakarta.persistence-api:3.0.0' - api 'io.seqera:lib-mail:1.1.0' - api 'io.seqera:wave-api:0.13.3' - api 'io.seqera:wave-utils:0.14.1' + api 'io.seqera:lib-mail:1.2.0' + api 'io.seqera:wave-api:0.14.0' + api 'io.seqera:wave-utils:0.15.0' implementation 'io.micronaut:micronaut-http-client' implementation 'io.micronaut:micronaut-jackson-databind' implementation 'io.micronaut.groovy:micronaut-runtime-groovy' @@ -55,14 +55,14 @@ dependencies { implementation 'io.micronaut.reactor:micronaut-reactor-http-client' implementation('io.seqera:tower-crypto:22.4.0-watson') { transitive = false } // to be replaced with 22.4.0 once released implementation 'org.apache.commons:commons-compress:1.27.1' - implementation 'org.apache.commons:commons-lang3:3.12.0' + implementation 'org.apache.commons:commons-lang3:3.17.0' implementation 'io.kubernetes:client-java:19.0.0' implementation 'io.kubernetes:client-java-api-fluent:18.0.1' implementation 'com.google.code.gson:gson:2.9.0' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'com.squareup.moshi:moshi:1.14.0' - implementation 'com.squareup.moshi:moshi-adapters:1.14.0' + implementation 'com.squareup.moshi:moshi:1.15.1' + implementation 'com.squareup.moshi:moshi-adapters:1.15.1' implementation 'redis.clients:jedis:5.1.3' implementation 'io.github.resilience4j:resilience4j-ratelimiter:0.17.0' implementation 'io.micronaut:micronaut-retry' diff --git a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle index 6763c4926..835199bd4 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle @@ -40,7 +40,7 @@ java { } dependencies { - implementation 'org.slf4j:slf4j-api:2.0.13' + implementation 'org.slf4j:slf4j-api:2.0.16' testImplementation 'ch.qos.logback:logback-core:1.5.12' testImplementation 'ch.qos.logback:logback-classic:1.5.12' From 9ba433ce6f9ab68b6c13f6719d9116840217a0fd Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 16 Nov 2024 18:14:23 +0100 Subject: [PATCH 47/54] Bump Netty version 4.1.115.Final Signed-off-by: Paolo Di Tommaso --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index ccbc223b5..7de40d237 100644 --- a/build.gradle +++ b/build.gradle @@ -103,6 +103,7 @@ dependencies { // upgrade indirect dependencies runtimeOnly 'org.bouncycastle:bcpkix-jdk18on:1.78' runtimeOnly 'org.bitbucket.b_c:jose4j:0.9.4' + runtimeOnly 'io.netty:netty-bom:4.1.115.Final' } application { From 328e9ea36238af533c316c1868fb8ab7960bca3d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 17 Nov 2024 21:07:53 +0100 Subject: [PATCH 48/54] Bump guava to version 33.3.1-jre Signed-off-by: Paolo Di Tommaso --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7de40d237..289bb5c01 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,7 @@ dependencies { implementation 'io.micronaut:micronaut-websocket' implementation 'org.apache.groovy:groovy-json' implementation 'org.apache.groovy:groovy-nio' - implementation 'com.google.guava:guava:32.1.2-jre' + implementation 'com.google.guava:guava:33.3.1-jre' implementation 'dev.failsafe:failsafe:3.1.0' implementation 'io.micronaut.reactor:micronaut-reactor' implementation 'io.micronaut.reactor:micronaut-reactor-http-client' From ffd0dacd14dc365cabb23a35a9cac4a0f06d37ae Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 17 Nov 2024 18:55:27 +0100 Subject: [PATCH 49/54] Use runAsync instead supplyAsync Signed-off-by: Paolo Di Tommaso --- .../io/seqera/wave/filter/PullMetricsRequestsFilter.groovy | 4 ++-- .../service/builder/impl/ContainerBuildServiceImpl.groovy | 3 ++- .../wave/service/mirror/ContainerMirrorServiceImpl.groovy | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy index da19fc7c6..bb70548ac 100644 --- a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy +++ b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy @@ -83,10 +83,10 @@ class PullMetricsRequestsFilter implements HttpServerFilter { final contentType = response.headers.get(HttpHeaders.CONTENT_TYPE) if( contentType && contentType in MANIFEST_TYPES ) { final route = routeHelper.parse(request.path) - CompletableFuture.supplyAsync(() -> metricsService.incrementPullsCounter(route.identity), executor) + CompletableFuture.runAsync(() -> metricsService.incrementPullsCounter(route.identity), executor) final version = route.request?.containerConfig?.fusionVersion() if (version) { - CompletableFuture.supplyAsync(() -> metricsService.incrementFusionPullsCounter(route.identity), executor) + CompletableFuture.runAsync(() -> metricsService.incrementFusionPullsCounter(route.identity), executor) } } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy index 6dbe9f787..895a90210 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy @@ -206,7 +206,8 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler metricsService.incrementBuildsCounter(request.identity), executor) + CompletableFuture + .runAsync(() -> metricsService.incrementBuildsCounter(request.identity), executor) // launch the build async CompletableFuture diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy index 42c4bafce..1126fe87d 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy @@ -75,7 +75,7 @@ class ContainerMirrorServiceImpl implements ContainerMirrorService, JobHandler metricsService.incrementMirrorsCounter(request.identity), ioExecutor) + CompletableFuture.runAsync(() -> metricsService.incrementMirrorsCounter(request.identity), ioExecutor) jobService.launchMirror(request) return new BuildTrack(request.mirrorId, request.targetImage, false, null) } From 7af3046fef4ce5880a8cf9454dab7bc5f4d0344b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 17 Nov 2024 17:35:39 +0100 Subject: [PATCH 50/54] Remove deprecated ThreadPoolBuilder Signed-off-by: Paolo Di Tommaso --- .../seqera/wave/util/ThreadPoolBuilder.groovy | 176 ------------------ 1 file changed, 176 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy diff --git a/src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy b/src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy deleted file mode 100644 index 7afdf0e06..000000000 --- a/src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.util - - - -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.RejectedExecutionHandler -import java.util.concurrent.ThreadFactory -import java.util.concurrent.ThreadPoolExecutor -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -/** - * Builder class to create instance of {@link ThreadPoolExecutor} - * - * @author Paolo Di Tommaso - */ -@Slf4j -@CompileStatic -@Deprecated -class ThreadPoolBuilder { - - static AtomicInteger poolCount = new AtomicInteger() - - private String name - - private int minSize - - private int maxSize - - private BlockingQueue workQueue - - private int queueSize = -1 - - private Long keepAliveTime - - private RejectedExecutionHandler rejectionPolicy - - private ThreadFactory threadFactory - - private boolean allowCoreThreadTimeout - - String getName() { name } - - int getMinSize() { minSize } - - int getMaxSize() { maxSize } - - int getQueueSize() { queueSize } - - BlockingQueue getWorkQueue() { workQueue } - - Long getKeepAliveTime() { keepAliveTime } - - RejectedExecutionHandler getRejectionPolicy() { rejectionPolicy } - - ThreadFactory getThreadFactory() { threadFactory } - - boolean getAllowCoreThreadTimeout() { allowCoreThreadTimeout } - - ThreadPoolBuilder withName(String name) { - if( name ) { - this.name = name - this.threadFactory = new CustomThreadFactory(name) - } - return this - } - - ThreadPoolBuilder withThreadFactory(ThreadFactory threadFactory) { - assert !name || !threadFactory, "Property 'threadFactory' or 'name' was already set" - this.threadFactory = threadFactory - return this - } - - ThreadPoolBuilder withRejectionPolicy(RejectedExecutionHandler rejectionPolicy) { - this.rejectionPolicy = rejectionPolicy - return this - } - - ThreadPoolBuilder withMinSize(int min) { - this.minSize = min - return this - } - - ThreadPoolBuilder withMaxSize(int max) { - this.maxSize = max - return this - } - - ThreadPoolBuilder withQueueSize(int size) { - this.queueSize = size - this.workQueue = new LinkedBlockingQueue(size) - return this - } - - ThreadPoolBuilder withQueue(BlockingQueue workQueue) { - this.workQueue = workQueue - return this - } - - ThreadPoolBuilder withKeepAliveTime( long millis ) { - keepAliveTime = millis - return this - } - - ThreadPoolBuilder withAllowCoreThreadTimeout(boolean flag) { - this.allowCoreThreadTimeout = flag - return this - } - - ThreadPoolExecutor build() { - assert minSize <= maxSize - - if( !name ) - name = "nf-thread-pool-${poolCount.getAndIncrement()}" - - if(keepAliveTime==null) - keepAliveTime = 60_000 - if( workQueue==null ) - workQueue = new LinkedBlockingQueue<>() - if( rejectionPolicy==null ) - rejectionPolicy = new ThreadPoolExecutor.CallerRunsPolicy() - if( threadFactory==null ) - threadFactory = new CustomThreadFactory(name) - - log.debug "Creating thread pool '$name' minSize=$minSize; maxSize=$maxSize; workQueue=${workQueue.getClass().getSimpleName()}[${queueSize}]; allowCoreThreadTimeout=$allowCoreThreadTimeout" - - final result = new ThreadPoolExecutor( - minSize, - maxSize, - keepAliveTime, TimeUnit.MILLISECONDS, - workQueue, - threadFactory, - rejectionPolicy) - - result.allowCoreThreadTimeOut(allowCoreThreadTimeout) - - return result - } - - - static ThreadPoolExecutor io(String name=null) { - io(10, 100, 10_000, name) - } - - - static ThreadPoolExecutor io(int min, int max, int queue, String name=null) { - new ThreadPoolBuilder() - .withMinSize(min) - .withMaxSize(max) - .withQueueSize(queue) - .withName(name) - .build() - } - -} From cf813e0a8e0f6a0273edc0a6f3f5d051b0bc911f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 17 Nov 2024 22:14:36 +0100 Subject: [PATCH 51/54] Replace Guava cache with Caffeine (#745) Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- .../seqera/wave/auth/RegistryAuthServiceImpl.groovy | 13 ++++++------- .../wave/auth/RegistryLookupServiceImpl.groovy | 13 ++++++------- .../io/seqera/wave/service/aws/AwsEcrService.groovy | 8 ++++---- .../service/data/queue/AbstractMessageQueue.groovy | 6 +++--- .../tower/client/connector/TowerConnector.groovy | 10 +++++----- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/VERSION b/VERSION index 63e799cf4..0b963338f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.1 +1.14.2-B3 diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index d8ea12d26..00dc5e9fa 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -21,13 +21,12 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse import java.time.Duration -import java.util.concurrent.ExecutionException +import java.util.concurrent.CompletionException import java.util.concurrent.TimeUnit -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache -import com.google.common.util.concurrent.UncheckedExecutionException +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine +import com.github.benmanes.caffeine.cache.LoadingCache import groovy.json.JsonSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic @@ -101,7 +100,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { return result } - private LoadingCache cacheTokens = CacheBuilder + private LoadingCache cacheTokens = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) @@ -271,7 +270,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { try { return cacheTokens.get(key) } - catch (UncheckedExecutionException | ExecutionException e) { + catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup // and throws the causing exception that should be `RegistryUnauthorizedAccessException` throw e.cause diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 0dc641879..58c41f617 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -20,13 +20,12 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse -import java.util.concurrent.ExecutionException +import java.util.concurrent.CompletionException import java.util.concurrent.TimeUnit -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache -import com.google.common.util.concurrent.UncheckedExecutionException +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine +import com.github.benmanes.caffeine.cache.LoadingCache import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.configuration.HttpClientConfig @@ -74,7 +73,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService { } } - private LoadingCache cache = CacheBuilder + private LoadingCache cache = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterAccess(1, TimeUnit.HOURS) @@ -120,7 +119,7 @@ class RegistryLookupServiceImpl implements RegistryLookupService { final auth = cache.get(endpoint) return new RegistryInfo(registry, endpoint, auth) } - catch (UncheckedExecutionException | ExecutionException e) { + catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup // and throws the causing exception that should be `RegistryUnauthorizedAccessException` throw e.cause diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 17fdb3720..a85b1b354 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -21,9 +21,9 @@ package io.seqera.wave.service.aws import java.util.concurrent.TimeUnit import java.util.regex.Pattern -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine +import com.github.benmanes.caffeine.cache.LoadingCache import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -73,7 +73,7 @@ class AwsEcrService { } } - private LoadingCache cache = CacheBuilder + private LoadingCache cache = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterWrite(3, TimeUnit.HOURS) diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy index 0f6ddc9a0..3a90e3108 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy @@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger -import com.google.common.cache.Cache -import com.google.common.cache.CacheBuilder +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.websocket.exceptions.WebSocketSessionException @@ -60,7 +60,7 @@ abstract class AbstractMessageQueue implements Runnable { final private String name0 - final private Cache closedClients = CacheBuilder + final private Cache closedClients = Caffeine.newBuilder() .newBuilder() .expireAfterWrite(10, TimeUnit.MINUTES) .build() diff --git a/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy b/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy index ec213628a..e4fa503ec 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy @@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.function.Function -import com.google.common.cache.Cache -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine +import com.github.benmanes.caffeine.cache.LoadingCache import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -92,7 +92,7 @@ abstract class TowerConnector { } } - private LoadingCache> refreshCache = CacheBuilder> + private LoadingCache> refreshCache = Caffeine.newBuilder() .newBuilder() .expireAfterWrite(1, TimeUnit.MINUTES) .build(loader) From aaf0420cdc1c224cbcacc1283757a8aa5f2d856b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 18 Nov 2024 07:46:09 +0100 Subject: [PATCH 52/54] Migration to virtual threads - phase 1 (#746) Signed-off-by: Paolo Di Tommaso --- build.gradle | 3 +-- .../wave/auth/RegistryAuthServiceImpl.groovy | 13 ++++++++----- .../wave/auth/RegistryLookupServiceImpl.groovy | 10 ++++++---- .../wave/controller/BuildController.groovy | 2 +- .../wave/controller/ContainerController.groovy | 6 +++--- .../wave/controller/InspectController.groovy | 2 +- .../wave/controller/MetricsController.groovy | 2 +- .../wave/controller/MirrorController.groovy | 2 +- .../controller/RegistryProxyController.groovy | 2 +- .../seqera/wave/controller/ScanController.groovy | 2 +- .../wave/controller/ServiceInfoController.groovy | 2 +- .../wave/controller/ValidateController.groovy | 2 +- .../seqera/wave/controller/ViewController.groovy | 2 +- .../seqera/wave/core/ContainerAugmenter.groovy | 3 +-- .../seqera/wave/core/RegistryProxyService.groovy | 13 +++++++++++-- .../io/seqera/wave/http/HttpClientFactory.groovy | 15 +++++++++++---- .../seqera/wave/service/aws/AwsEcrService.groovy | 10 ++++++---- .../data/queue/AbstractMessageQueue.groovy | 13 ++++++++----- .../data/stream/AbstractMessageStream.groovy | 4 ++++ .../io/seqera/wave/service/job/JobManager.groovy | 13 +++++++++---- .../pairing/socket/PairingWebSocket.groovy | 2 +- .../tower/client/connector/TowerConnector.groovy | 16 ++++++++++------ .../connector/WebSocketTowerConnector.groovy | 9 +-------- .../wave/service/job/JobManagerTest.groovy | 10 +++++----- 24 files changed, 94 insertions(+), 64 deletions(-) diff --git a/build.gradle b/build.gradle index 289bb5c01..56c9eb999 100644 --- a/build.gradle +++ b/build.gradle @@ -156,8 +156,7 @@ jib { run{ def envs = findProperty('micronautEnvs') - // note: "--enable-preview" is required to use virtual threads on Java 19 and 20 - def args = ["-Dmicronaut.environments=$envs","--enable-preview"] + def args = ["-Dmicronaut.environments=$envs","-Djdk.tracePinnedThreads=short"] if( environment['JVM_OPTS'] ) args.add(environment['JVM_OPTS']) jvmArgs args systemProperties 'DOCKER_USER': project.findProperty('DOCKER_USER') ?: environment['DOCKER_USER'], diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index 00dc5e9fa..7ae087db7 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -24,9 +24,9 @@ import java.time.Duration import java.util.concurrent.CompletionException import java.util.concurrent.TimeUnit +import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine -import com.github.benmanes.caffeine.cache.LoadingCache import groovy.json.JsonSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic @@ -100,11 +100,12 @@ class RegistryAuthServiceImpl implements RegistryAuthService { return result } - private LoadingCache cacheTokens = Caffeine.newBuilder() + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cacheTokens = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) - .build(loader) + .buildAsync(loader) @Inject private RegistryLookupService lookupService @@ -268,7 +269,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - return cacheTokens.get(key) + // FIXME https://github.com/seqeralabs/wave/issues/747 + return cacheTokens.synchronous().get(key) } catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup @@ -286,7 +288,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { */ void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) - cacheTokens.invalidate(key) + // FIXME https://github.com/seqeralabs/wave/issues/747 + cacheTokens.synchronous().invalidate(key) tokenStore.remove(getStableKey(key)) } diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 58c41f617..2b15b8efa 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -23,9 +23,9 @@ import java.net.http.HttpResponse import java.util.concurrent.CompletionException import java.util.concurrent.TimeUnit +import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine -import com.github.benmanes.caffeine.cache.LoadingCache import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.configuration.HttpClientConfig @@ -73,11 +73,12 @@ class RegistryLookupServiceImpl implements RegistryLookupService { } } - private LoadingCache cache = Caffeine.newBuilder() + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cache = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterAccess(1, TimeUnit.HOURS) - .build(loader) + .buildAsync(loader) protected RegistryAuth lookup0(URI endpoint) { final httpClient = HttpClientFactory.followRedirectsHttpClient() @@ -116,7 +117,8 @@ class RegistryLookupServiceImpl implements RegistryLookupService { RegistryInfo lookup(String registry) { try { final endpoint = registryEndpoint(registry) - final auth = cache.get(endpoint) + // FIXME https://github.com/seqeralabs/wave/issues/747 + final auth = cache.synchronous().get(endpoint) return new RegistryInfo(registry, endpoint, auth) } catch (CompletionException e) { diff --git a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy index 06679fa4a..f99549087 100644 --- a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy @@ -42,7 +42,7 @@ import jakarta.inject.Inject @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class BuildController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 10f71ddb4..a65568c40 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -103,7 +103,7 @@ import static java.util.concurrent.CompletableFuture.completedFuture @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ContainerController { @Inject @@ -181,13 +181,13 @@ class ContainerController { @Deprecated @Post('/container-token') - @ExecuteOn(TaskExecutors.IO) + @ExecuteOn(TaskExecutors.BLOCKING) CompletableFuture> getToken(HttpRequest httpRequest, @Body SubmitContainerTokenRequest req) { return getContainerImpl(httpRequest, req, false) } @Post('/v1alpha2/container') - @ExecuteOn(TaskExecutors.IO) + @ExecuteOn(TaskExecutors.BLOCKING) CompletableFuture> getTokenV2(HttpRequest httpRequest, @Body SubmitContainerTokenRequest req) { return getContainerImpl(httpRequest, req, true) } diff --git a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy index 35e50bc58..6093d4dfb 100644 --- a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy @@ -50,7 +50,7 @@ import static io.seqera.wave.util.ContainerHelper.patchPlatformEndpoint @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class InspectController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy index a9ce6f89a..f8008674e 100644 --- a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy @@ -52,7 +52,7 @@ import static io.micronaut.http.HttpHeaders.WWW_AUTHENTICATE @Requires(property = 'wave.metrics.enabled', value = 'true') @Secured(SecurityRule.IS_AUTHENTICATED) @Controller -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class MetricsController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy index 8aaf87a51..c9c3bc8f7 100644 --- a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy @@ -36,7 +36,7 @@ import jakarta.inject.Inject @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class MirrorController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy index ec6416537..263763299 100644 --- a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy @@ -67,7 +67,7 @@ import reactor.core.publisher.Mono @Slf4j @CompileStatic @Controller("/v2") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class RegistryProxyController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy index 230a8b4a9..43fc38a5d 100644 --- a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy @@ -39,7 +39,7 @@ import jakarta.inject.Inject @CompileStatic @Requires(bean = ContainerScanService) @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ScanController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy b/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy index 207c6e8d5..7f0a895e4 100644 --- a/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy @@ -39,7 +39,7 @@ import io.seqera.wave.util.BuildInfo @Slf4j @Controller("/") @CompileStatic -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ServiceInfoController { @Value('${wave.landing.url}') diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy index f7136f572..fd23fbdc6 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy @@ -26,8 +26,8 @@ import io.seqera.wave.auth.RegistryAuthService import jakarta.inject.Inject import jakarta.validation.Valid -@ExecuteOn(TaskExecutors.IO) @Controller("/validate-creds") +@ExecuteOn(TaskExecutors.BLOCKING) class ValidateController { @Inject RegistryAuthService loginService diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index d30db91d8..68328eb36 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -59,7 +59,7 @@ import static io.seqera.wave.util.DataTimeUtils.formatTimestamp @Slf4j @CompileStatic @Controller("/view") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ViewController { @Inject diff --git a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy index 909cc8d5d..bd3837e87 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy @@ -280,7 +280,7 @@ class ContainerAugmenter { return result } - synchronized protected Map layerBlob(String image, ContainerLayer layer) { + protected Map layerBlob(String image, ContainerLayer layer) { log.debug "Adding layer: $layer to image: $client.registry.name/$image" // store the layer blob in the cache final String path = "$client.registry.name/v2/$image/blobs/$layer.gzipDigest" @@ -295,7 +295,6 @@ class ContainerAugmenter { protected Tuple2 updateImageManifest(String imageName, String imageManifest, String newImageConfigDigest, newImageConfigSize, boolean oci) { - // turn the json string into a json map // and append the new layer final manifest = (Map) new JsonSlurper().parseText(imageManifest) diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index 32cf49604..01ed6bd94 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.core +import java.util.concurrent.CompletableFuture + import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j @@ -193,7 +195,7 @@ class RegistryProxyService { String getImageDigest(String containerImage, PlatformId identity, boolean retryOnNotFound=false) { try { - return getImageDigest0(containerImage, identity, retryOnNotFound) + return getImageDigest0(containerImage, identity, retryOnNotFound).get() } catch(Exception e) { log.warn "Unable to retrieve digest for image '${containerImage}' -- cause: ${e.message}" @@ -203,8 +205,15 @@ class RegistryProxyService { static private List RETRY_ON_NOT_FOUND = HTTP_RETRYABLE_ERRORS + 404 + // note: return a CompletableFuture to force micronaut to use caffeine AsyncCache + // that provides a workaround about the use of virtual threads with SyncCache + // see https://github.com/ben-manes/caffeine/issues/1468#issuecomment-1906733926 @Cacheable(value = 'cache-registry-proxy', atomic = true, parameters = ['image']) - protected String getImageDigest0(String image, PlatformId identity, boolean retryOnNotFound) { + protected CompletableFuture getImageDigest0(String image, PlatformId identity, boolean retryOnNotFound) { + CompletableFuture.completedFuture(getImageDigest1(image, identity, retryOnNotFound)) + } + + protected String getImageDigest1(String image, PlatformId identity, boolean retryOnNotFound) { final coords = ContainerCoordinates.parse(image) final route = RoutePath.v2manifestPath(coords, identity) final proxyClient = client(route) diff --git a/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy b/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy index f1301037e..46c9730d2 100644 --- a/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy +++ b/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy @@ -22,6 +22,7 @@ import java.net.http.HttpClient import java.time.Duration import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.locks.ReentrantLock import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -39,9 +40,9 @@ class HttpClientFactory { static private Duration timeout = Duration.ofSeconds(20) - static private final Object l1 = new Object() + static private final ReentrantLock l1 = new ReentrantLock() - static private final Object l2 = new Object() + static private final ReentrantLock l2 = new ReentrantLock() private static HttpClient client1 @@ -51,20 +52,26 @@ class HttpClientFactory { static HttpClient followRedirectsHttpClient() { if( client1!=null ) return client1 - synchronized (l1) { + l1.lock() + try { if( client1!=null ) return client1 return client1=followRedirectsHttpClient0() + } finally { + l1.unlock() } } static HttpClient neverRedirectsHttpClient() { if( client2!=null ) return client2 - synchronized (l2) { + l2.lock() + try { if( client2!=null ) return client2 return client2=neverRedirectsHttpClient0() + } finally { + l2.unlock() } } diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index a85b1b354..30f9d0e04 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -21,9 +21,9 @@ package io.seqera.wave.service.aws import java.util.concurrent.TimeUnit import java.util.regex.Pattern +import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine -import com.github.benmanes.caffeine.cache.LoadingCache import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -73,11 +73,12 @@ class AwsEcrService { } } - private LoadingCache cache = Caffeine.newBuilder() + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cache = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterWrite(3, TimeUnit.HOURS) - .build(loader) + .buildAsync(loader) private EcrClient ecrClient(String accessKey, String secretKey, String region) { @@ -126,7 +127,8 @@ class AwsEcrService { try { // get the token from the cache, if missing the it's automatically // fetch using the AWS ECR client - return cache.get(new AwsCreds(accessKey,secretKey,region,isPublic)) + // FIXME https://github.com/seqeralabs/wave/issues/747 + return cache.synchronous().get(new AwsCreds(accessKey,secretKey,region,isPublic)) } catch (Exception e) { final type = isPublic ? "ECR public" : "ECR" diff --git a/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy b/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy index 3a90e3108..8c2d3fa2c 100644 --- a/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/queue/AbstractMessageQueue.groovy @@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger -import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.AsyncCache import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -60,10 +60,11 @@ abstract class AbstractMessageQueue implements Runnable { final private String name0 - final private Cache closedClients = Caffeine.newBuilder() + // FIXME https://github.com/seqeralabs/wave/issues/747 + final private AsyncCache closedClients = Caffeine.newBuilder() .newBuilder() .expireAfterWrite(10, TimeUnit.MINUTES) - .build() + .buildAsync() AbstractMessageQueue(MessageQueue broker) { final type = TypeHelper.getGenericType(this, 0) @@ -149,13 +150,15 @@ abstract class AbstractMessageQueue implements Runnable { @Override void run() { + // FIXME https://github.com/seqeralabs/wave/issues/747 + final clientsCache = closedClients.synchronous() while( !thread.isInterrupted() ) { try { int sent=0 final clients = new HashMap>(this.clients) for( Map.Entry> entry : clients ) { // ignore clients marked as closed - if( closedClients.getIfPresent(entry.key)) + if( clientsCache.getIfPresent(entry.key)) continue // infer the target queue from the client key final target = targetFromClientKey(entry.key) @@ -173,7 +176,7 @@ abstract class AbstractMessageQueue implements Runnable { // offer back the value to be processed again broker.offer(target, value) if( e.message?.contains('close') ) { - closedClients.put(entry.key, true) + clientsCache.put(entry.key, true) } } } diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy index 560a81306..ae16d7140 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy @@ -98,6 +98,10 @@ abstract class AbstractMessageStream implements Closeable { * The {@link Predicate} to be invoked when a stream message is consumed (read from) the stream. */ void addConsumer(String streamId, MessageConsumer consumer) { + // the use of synchronized block is meant to prevent a race condition while + // updating the 'listeners' from concurrent invocations. + // however, considering the addConsumer is invoked during the initialization phase + // (and therefore in the same thread) in should not be really needed. synchronized (listeners) { if( listeners.containsKey(streamId)) throw new IllegalStateException("Only one consumer can be defined for each stream - offending streamId=$streamId; consumer=$consumer") diff --git a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy index 1c4ec6c03..e10cde8d4 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy @@ -21,6 +21,7 @@ package io.seqera.wave.service.job import java.time.Duration import java.time.Instant +import com.github.benmanes.caffeine.cache.AsyncCache import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic @@ -50,16 +51,19 @@ class JobManager { @Inject private JobConfig config - private Cache debounceCache + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncCache debounceCache @PostConstruct void init() { log.info "Creating job manager - config=$config" - debounceCache = Caffeine.newBuilder().expireAfterWrite(config.graceInterval.multipliedBy(2)).build() + debounceCache = Caffeine + .newBuilder() + .expireAfterWrite(config.graceInterval.multipliedBy(2)) + .buildAsync() queue.addConsumer((job)-> processJob(job)) } - protected boolean processJob(JobSpec jobSpec) { try { return processJob0(jobSpec) @@ -73,7 +77,8 @@ class JobManager { } protected JobState state(JobSpec job) { - return state0(job, config.graceInterval, debounceCache) + // FIXME https://github.com/seqeralabs/wave/issues/747 + return state0(job, config.graceInterval, debounceCache.synchronous()) } protected JobState state0(final JobSpec job, final Duration graceInterval, final Cache cache) { diff --git a/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy b/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy index a4f7b0785..3e6682fa5 100644 --- a/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy +++ b/src/main/groovy/io/seqera/wave/service/pairing/socket/PairingWebSocket.groovy @@ -49,7 +49,7 @@ import static io.seqera.wave.util.LongRndKey.rndHex @Slf4j @CompileStatic @Singleton -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) @ServerWebSocket("/pairing/{service}/token/{token}{?endpoint}") class PairingWebSocket { diff --git a/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy b/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy index e4fa503ec..27e12d87f 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy @@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.function.Function +import com.github.benmanes.caffeine.cache.AsyncLoadingCache import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.CacheLoader import com.github.benmanes.caffeine.cache.Caffeine -import com.github.benmanes.caffeine.cache.LoadingCache import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -82,7 +82,7 @@ abstract class TowerConnector { private SpillwayRateLimiter limiter @Inject - @Named(TaskExecutors.IO) + @Named(TaskExecutors.BLOCKING) private volatile ExecutorService ioExecutor private CacheLoader> loader = new CacheLoader>() { @@ -92,14 +92,18 @@ abstract class TowerConnector { } } - private LoadingCache> refreshCache = Caffeine.newBuilder() + private AsyncLoadingCache> refreshCache = Caffeine .newBuilder() .expireAfterWrite(1, TimeUnit.MINUTES) - .build(loader) + .buildAsync(loader) /** Only for testing - do not use */ Cache> refreshCache0() { - return refreshCache + return refreshCache.synchronous() + } + + protected ExecutorService getIoExecutor() { + return ioExecutor } /** @@ -242,7 +246,7 @@ abstract class TowerConnector { * @return The refreshed {@link JwtAuth} object */ protected CompletableFuture refreshJwtToken(String endpoint, JwtAuth auth) { - return refreshCache.get(new JwtRefreshParams(endpoint,auth)) + return refreshCache.synchronous().get(new JwtRefreshParams(endpoint,auth)) } protected CompletableFuture refreshJwtToken0(String endpoint, JwtAuth auth) { diff --git a/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy b/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy index c52646638..04c591a09 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy @@ -19,19 +19,16 @@ package io.seqera.wave.tower.client.connector import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService import java.util.function.Function import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires -import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.pairing.socket.msg.PairingMessage import io.seqera.wave.service.pairing.socket.msg.ProxyHttpRequest import io.seqera.wave.service.pairing.socket.msg.ProxyHttpResponse import jakarta.inject.Inject -import jakarta.inject.Named import jakarta.inject.Singleton import static io.seqera.wave.service.pairing.PairingService.TOWER_SERVICE /** @@ -49,15 +46,11 @@ class WebSocketTowerConnector extends TowerConnector { @Inject private PairingChannel channel - @Inject - @Named(TaskExecutors.IO) - private ExecutorService ioExecutor - @Override CompletableFuture sendAsync(String endpoint, ProxyHttpRequest request) { return channel .sendRequest(TOWER_SERVICE, endpoint, request) - .thenApplyAsync(Function.identity() as Function, ioExecutor) + .thenApplyAsync(Function.identity() as Function, getIoExecutor()) } } diff --git a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy index f58513a13..04ea45d71 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy @@ -26,7 +26,6 @@ import java.time.Instant import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine - /** * * @author Munish Chouhan @@ -38,7 +37,7 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def cache = Caffeine.newBuilder().build() + def cache = Caffeine.newBuilder().buildAsync() def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now(), Duration.ofMinutes(10)) @@ -57,7 +56,8 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config) + def cache = Caffeine.newBuilder().buildAsync() + def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now(), Duration.ofMinutes(10)) @@ -75,7 +75,7 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def cache = Caffeine.newBuilder().build() + def cache = Caffeine.newBuilder().buildAsync() def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config:config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now() - Duration.ofMinutes(5), Duration.ofMinutes(2)) @@ -94,7 +94,7 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def cache = Caffeine.newBuilder().build() + def cache = Caffeine.newBuilder().buildAsync() def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now().minus(Duration.ofMillis(500)), Duration.ofMinutes(10)) From ec9ae8334d19528ca068b2c13c725ec86e5319b9 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 18 Nov 2024 08:03:54 +0100 Subject: [PATCH 53/54] [release] bump version 1.15.0 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- changelog.txt | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0b963338f..141f2e805 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.2-B3 +1.15.0 diff --git a/changelog.txt b/changelog.txt index 7190df430..5e5b7fdad 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,14 @@ # Wave changelog +1.15.0 - 18 Nov 2024 +- Migration to virtual threads - phase 1 (#746) [aaf0420c] +- Use runAsync instead supplyAsync [ffd0dacd] +- Remove deprecated ThreadPoolBuilder [7af3046f] +- Replace Guava cache with Caffeine (#745) [cf813e0a] +- Update project deps [f24b684d] +- Bump guava to version 33.3.1-jre [328e9ea3] +- Bump Netty version 4.1.115.Final [9ba433ce] +- Bump gradle 8.10.2 [52272fe1] + 1.14.1 - 14 Nov 2024 - Fix creds validation endpoint (#740) [8c0f3a4c] From 080d5cce435ba3b6986ea175545b5b165b370add Mon Sep 17 00:00:00 2001 From: Justine Geffen Date: Mon, 18 Nov 2024 15:30:19 +0200 Subject: [PATCH 54/54] Switched env vars around (#748) Switched env vars around for accuracy. --- docs/cli/index.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index 6581f628d..45f1239e2 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -27,8 +27,8 @@ The following CLI arguments are available for Seqera Platform integration: The following environment variables are available for Seqera Platform integration: -- `TOWER_API_ENDPOINT`: A Seqera Platform auth token so that Wave can access your private registry credentials. -- `TOWER_ACCESS_TOKEN`: For Enterprise customers, the URL endpoint for your instance, such as `https://api.cloud.seqera.io`. +- `TOWER_ACCESS_TOKEN`: A Seqera Platform auth token so that Wave can access your private registry credentials. +- `TOWER_API_ENDPOINT`: For Enterprise customers, the URL endpoint for your instance, such as `https://api.cloud.seqera.io`. - `TOWER_WORKSPACE_ID`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. ## Usage limits