diff --git a/build.gradle b/build.gradle index 622c954b8..cd9ab92e0 100644 --- a/build.gradle +++ b/build.gradle @@ -94,6 +94,8 @@ dependencies { implementation "io.micronaut:micronaut-management" //views implementation("io.micronaut.views:micronaut-views-handlebars") + //mongodb + implementation "io.micronaut.mongodb:micronaut-mongo-sync" } application { diff --git a/src/main/groovy/io/seqera/wave/configuration/MongoDBConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/MongoDBConfig.groovy new file mode 100644 index 000000000..be09fadd0 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/configuration/MongoDBConfig.groovy @@ -0,0 +1,47 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.configuration + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Value +import jakarta.inject.Singleton +/** + * MongoDB configuration + * + * @author Munish Chouhan + */ +@CompileStatic +@Singleton +@Slf4j +class MongoDBConfig { + + /** + * MongoDB database name + */ + @Value('${mongodb.database.name}') + String databaseName + + /** + * MongoDB uri + */ + @Value('${mongodb.uri}') + String uri + +} diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index fcfa3d7d7..b8455c2cc 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -259,7 +259,7 @@ class ContainerController { protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequestData data, TokenData token, String target, String ip) { try { - final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration) + final recrd = new WaveContainerRecord(req, token, data, target, ip) persistenceService.saveContainerRequest(token.value, recrd) } catch (Throwable e) { diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 10107f5d0..9555f3e55 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -21,12 +21,15 @@ package io.seqera.wave.service.persistence import java.time.Duration import java.time.Instant +import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import io.micronaut.data.annotation.MappedEntity import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.api.BuildStatusResponse +import jakarta.persistence.Id /** * A collection of request and response properties to be stored @@ -37,8 +40,11 @@ import io.seqera.wave.api.BuildStatusResponse @ToString @CompileStatic @EqualsAndHashCode +@MappedEntity +@Canonical class WaveBuildRecord { + @Id String buildId String dockerFile String condaFile diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index c6070f182..3373caf42 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -25,11 +25,13 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j +import io.micronaut.data.annotation.MappedEntity import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.api.FusionVersion import io.seqera.wave.api.SubmitContainerTokenRequest import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.token.TokenData import io.seqera.wave.tower.User +import jakarta.persistence.Id import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime /** * Model a Wave container request record @@ -40,8 +42,12 @@ import static io.seqera.wave.util.DataTimeUtils.parseOffsetDateTime @ToString(includeNames = true, includePackage = false) @Canonical @CompileStatic +@MappedEntity class WaveContainerRecord { + @Id + final String token; + /** * The Tower user associated with the request */ @@ -158,7 +164,8 @@ class WaveContainerRecord { */ final String fusionVersion - WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequestData data, String waveImage, String addr, Instant expiration) { + WaveContainerRecord(SubmitContainerTokenRequest request, TokenData token, ContainerRequestData data, String waveImage, String addr) { + this.token = token.value this.user = data.identity.user this.workspaceId = request.towerWorkspaceId this.containerImage = request.containerImage @@ -172,7 +179,7 @@ class WaveContainerRecord { this.containerFile = data.containerFile this.sourceImage = data.containerImage this.waveImage = waveImage - this.expiration = expiration + this.expiration = token.expiration this.ipAddress = addr final ts = parseOffsetDateTime(request.timestamp) ?: OffsetDateTime.now() this.timestamp = ts?.toInstant() @@ -184,6 +191,7 @@ class WaveContainerRecord { } WaveContainerRecord(WaveContainerRecord that, String sourceDigest, String waveDigest) { + this.token = that.token this.user = that.user this.workspaceId = that.workspaceId this.containerImage = that.containerImage diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index 871bbfff1..7a21981f0 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -21,13 +21,17 @@ package io.seqera.wave.service.persistence import java.time.Duration import java.time.Instant +import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import groovy.util.logging.Slf4j +import io.micronaut.data.annotation.MappedEntity import io.seqera.wave.service.scan.ScanResult import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.StringUtils +import jakarta.persistence.Id + /** * Model a Wave container scan result * @@ -37,7 +41,10 @@ import io.seqera.wave.util.StringUtils @ToString(includeNames = true, includePackage = false) @EqualsAndHashCode @CompileStatic +@MappedEntity +@Canonical class WaveScanRecord { + @Id String id String buildId Instant startTime diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/MongoDBPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/MongoDBPersistenceService.groovy new file mode 100644 index 000000000..9281c3407 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/MongoDBPersistenceService.groovy @@ -0,0 +1,149 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.persistence.impl + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoCollection +import com.mongodb.client.model.Filters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Primary +import io.micronaut.context.annotation.Requires +import io.seqera.wave.configuration.MongoDBConfig +import io.seqera.wave.core.ContainerDigestPair +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.persistence.WaveContainerRecord +import io.seqera.wave.service.persistence.WaveScanRecord +import jakarta.inject.Inject +import jakarta.inject.Singleton +import org.bson.Document +/** + * Implements a persistence service based based on SurrealDB + * + * @author : Munish Chouhan + */ +@Requires(env='mongodb') +@Primary +@Slf4j +@Singleton +@CompileStatic +class MongoDBPersistenceService implements PersistenceService { + + @Inject + private MongoClient mongoClient + + @Inject + private MongoDBConfig mongoDBConfig + + private final String WAVE_BUILD_COLLECTION = "wave_build" + private String WAVE_CONTAINER_COLLECTION = "wave_request" + private String WAVE_SCAN_COLLECTION = "wave_scan" + + @Override + void saveBuild(WaveBuildRecord build) { + try { + def collection = getCollection(WAVE_BUILD_COLLECTION, WaveBuildRecord.class) + collection.insertOne(build); + log.trace("Build request with id '{}' saved record: {}", build.getBuildId(), build); + } catch (Exception e) { + log.error("Error saving Build request record {}: {}\n{}", e.getMessage(), build, e); + } + } + + @Override + WaveBuildRecord loadBuild(String buildId) { + try { + def collection = getCollection(WAVE_BUILD_COLLECTION, WaveBuildRecord.class) + return collection.find(Filters.eq("_id", buildId)).first() + } catch (Exception e) { + log.error("Error fetching Build request record {}: {}", e.getMessage(), e); + } + return null + } + + @Override + WaveBuildRecord loadBuild(String targetImage, String digest) { + return null + } + + @Override + void saveContainerRequest(String token, WaveContainerRecord data) { + try { + def collection = getCollection(WAVE_CONTAINER_COLLECTION, WaveContainerRecord.class) + collection.insertOne(data); + log.trace("Container request with id '{}' saved record: {}", data.token, data); + } catch (Exception e) { + log.error("Error saving container request record {}: {}\n{}", e.getMessage(), data, e); + } + } + + @Override + void updateContainerRequest(String token, ContainerDigestPair digest) { + try { + def collection = getCollection(WAVE_CONTAINER_COLLECTION, WaveContainerRecord.class) + collection.updateOne(Filters.eq("token", token), new Document("\$set", new Document("sourceDigest", digest.source).append("waveDigest", digest.target))) + log.trace("Container request with id '{}' updated digest: {}", token, digest); + } catch (Exception e) { + log.error("Error updating Container request record {}: {}\n{}", e.getMessage(), token, e); + } + } + + @Override + WaveContainerRecord loadContainerRequest(String token) { + try { + def collection = getCollection(WAVE_CONTAINER_COLLECTION, WaveContainerRecord.class) + return collection.find(Filters.eq("_id", token)).first() + } catch (Exception e) { + log.error("Error fetching container request record {}: {}", e.getMessage(), e); + } + return null + } + + @Override + void createScanRecord(WaveScanRecord scanRecord) { + try { + def collection = getCollection(WAVE_SCAN_COLLECTION, WaveScanRecord.class) + collection.insertOne(scanRecord); + log.trace("Container scan with id '{}' saved record: {}", scanRecord.id, scanRecord); + } catch (Exception e) { + log.error("Error saving container scan record {}: {}\n{}", e.getMessage(), scanRecord, e); + } + } + + @Override + void updateScanRecord(WaveScanRecord scanRecord) { + + } + + @Override + WaveScanRecord loadScanRecord(String scanId) { + try { + def collection = getCollection(WAVE_SCAN_COLLECTION, WaveScanRecord.class) + return collection.find(Filters.eq("_id", scanId)).first() + } catch (Exception e) { + log.error("Error fetching container scan record {}: {}", e.getMessage(), e); + } + return null + } + + private MongoCollection getCollection(String collectionName, Class type) { + return mongoClient.getDatabase(mongoDBConfig.databaseName).getCollection(collectionName, type) + } +} diff --git a/src/main/resources/application-mongodb.yml b/src/main/resources/application-mongodb.yml new file mode 100644 index 000000000..b9cda3c38 --- /dev/null +++ b/src/main/resources/application-mongodb.yml @@ -0,0 +1,4 @@ +mongodb: + uri: mongodb://${MONGODB_USERNAME:''}:${MONGODB_PASSWORD:''}@${MONGODB_HOST:localhost}:${MONGODB_PORT:27017}/${MONGODB_NAME:wave} + database: + name: ${MONGODB_NAME:wave} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/MongoDBPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/MongoDBPersistenceServiceTest.groovy new file mode 100644 index 000000000..c8e3ae036 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/MongoDBPersistenceServiceTest.groovy @@ -0,0 +1,266 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.persistence.impl + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +import io.micronaut.context.ApplicationContext +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerLayer +import io.seqera.wave.api.SubmitContainerTokenRequest +import io.seqera.wave.core.ContainerDigestPair +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.ContainerRequestData +import io.seqera.wave.service.builder.BuildEvent +import io.seqera.wave.service.builder.BuildFormat +import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildResult +import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.persistence.WaveContainerRecord +import io.seqera.wave.service.persistence.WaveScanRecord +import io.seqera.wave.service.scan.ScanVulnerability +import io.seqera.wave.test.MongoDBTestContainer +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +/** + * + * @author Munish Chouhan + */ +class MongoDBPersistenceServiceTest extends Specification implements MongoDBTestContainer{ + + ApplicationContext applicationContext + + MongoDBPersistenceService persistence + + def setup() { + applicationContext = ApplicationContext.run([ + MONGODB_HOST : mongoDBHostName, + MONGODB_PORT : mongoDBPort + + ], 'test', 'mongodb') + persistence = applicationContext.getBean(MongoDBPersistenceService) + sleep(500) + } + def cleanup() { + applicationContext.close() + } + + + void "can insert an build"() { + given: + final String dockerFile = """\ + FROM quay.io/nextflow/bash + RUN echo "Look ma' building 🐳🐳 on the fly!" > /hello.txt + ENV NOW=${System.currentTimeMillis()} + """ + final String condaFile = """ + echo "Look ma' building 🐳🐳 on the fly!" > /hello.txt + """ + + final request = new BuildRequest( + 'container1234', + dockerFile, + condaFile, + null, + Path.of("."), + 'docker.io/my/repo:container1234', + PlatformId.NULL, + ContainerPlatform.of('amd64'), + 'docker.io/my/cache', + '127.0.0.1', + '{"config":"json"}', + null, + null, + 'scan12345', + null, + BuildFormat.DOCKER + ).withBuildId('1') + def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) + def event = new BuildEvent(request, result) + def build = WaveBuildRecord.fromEvent(event) + + when: + persistence.saveBuild(build) + then: + sleep 100 + def stored = persistence.loadBuild(request.buildId) + stored.buildId == request.buildId + stored.requestIp == '127.0.0.1' + } + + def 'should load a build record' () { + given: + final request = new BuildRequest( + 'container1234', + 'FROM foo:latest', + 'conda::recipe', + null, + Path.of("."), + 'docker.io/my/repo:container1234', + PlatformId.NULL, + ContainerPlatform.of('amd64'), + 'docker.io/my/cache', + '127.0.0.1', + '{"config":"json"}', + null, + null, + 'scan12345', + null, + BuildFormat.DOCKER + ).withBuildId('123') + def result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) + def event = new BuildEvent(request, result) + def record = WaveBuildRecord.fromEvent(event) + + and: + persistence.saveBuild(record) + + when: + def loaded = persistence.loadBuild(record.buildId) + + then: + loaded == record + } + + def 'should save and update a build' () { + given: + final request = new BuildRequest( + 'container1234', + 'FROM foo:latest', + 'conda::recipe', + null, + Path.of("/some/path"), + 'buildrepo:recipe-container1234', + PlatformId.NULL, + ContainerPlatform.of('amd64'), + 'docker.io/my/cache', + '127.0.0.1', + '{"config":"json"}', + null, + null, + 'scan12345', + null, + BuildFormat.DOCKER + ).withBuildId('123') + and: + def result = BuildResult.completed(request.buildId, 1, 'Hello', Instant.now().minusSeconds(60), 'xyz') + + and: + def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) + + when: + persistence.saveBuild(build1) + then: + persistence.loadBuild(request.buildId) == build1 + + } + + def 'should load a request record' () { + given: + def TOKEN = '123abc' + def cfg = new ContainerConfig(entrypoint: ['/opt/fusion'], + layers: [ new ContainerLayer(location: 'https://fusionfs.seqera.io/releases/v2.2.8-amd64.json')]) + def req = new SubmitContainerTokenRequest( + towerEndpoint: 'https://tower.nf', + towerWorkspaceId: 100, + containerConfig: cfg, + containerPlatform: ContainerPlatform.of('amd64'), + buildRepository: 'build.docker.io', + cacheRepository: 'cache.docker.io', + fingerprint: 'xyz', + timestamp: Instant.now().toString() + ) + def user = new User(id: 1, userName: 'foo', email: 'foo@gmail.com') + def data = new ContainerRequestData(new PlatformId(user,100), 'hello-world' ) + def wave = "wave.io/wt/$TOKEN/hello-world" + def addr = "100.200.300.400" + def exp = Instant.now().plusSeconds(3600) + and: + def request = new WaveContainerRecord(req, data, wave, addr, exp) + + and: + persistence.saveContainerRequest(TOKEN, request) + and: + sleep 200 // <-- the above request is async, give time to save it + + when: + def loaded = persistence.loadContainerRequest(TOKEN) + then: + loaded == request + + + // should update the record + when: + persistence.updateContainerRequest(TOKEN, new ContainerDigestPair('111', '222')) + and: + sleep 200 + then: + def updated = persistence.loadContainerRequest(TOKEN) + and: + updated.sourceDigest == '111' + updated.sourceImage == request.sourceImage + and: + updated.waveDigest == '222' + updated.waveImage == request.waveImage + + } + + def 'should save a scan and load a result' () { + given: + def NOW = Instant.now() + def SCAN_ID = 'a1' + def BUILD_ID = '100' + def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') + def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') + def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') + def scanRecord = new WaveScanRecord(SCAN_ID, BUILD_ID, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3]) + when: + persistence.createScanRecord(new WaveScanRecord(SCAN_ID, BUILD_ID, NOW)) + persistence.updateScanRecord(scanRecord) + then: + def result = persistence.loadScanRecord(SCAN_ID) + and: + result == scanRecord + and: + def scan = persistence.loadScanResult(SCAN_ID) + scan.status == 'SUCCEEDED' + scan.buildId == BUILD_ID + scan.vulnerabilities == scanRecord.vulnerabilities + + when: + def SCAN_ID2 = 'b2' + def BUILD_ID2 = '102' + def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1]) + and: + persistence.createScanRecord(new WaveScanRecord(SCAN_ID2, BUILD_ID2, NOW)) + // should save the same CVE into another build + persistence.updateScanRecord(scanRecord2) + then: + def result2 = persistence.loadScanRecord(SCAN_ID2) + and: + result2 == scanRecord2 + } + + +} diff --git a/src/test/groovy/io/seqera/wave/test/MongoDBTestContainer.groovy b/src/test/groovy/io/seqera/wave/test/MongoDBTestContainer.groovy new file mode 100644 index 000000000..e599b7407 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/test/MongoDBTestContainer.groovy @@ -0,0 +1,60 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.test + +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.utility.DockerImageName +/** + * trait for MongoDB test container + * + * @author Munish Chouhan + */ +trait MongoDBTestContainer{ + private static final Logger log = LoggerFactory.getLogger(MongoDBTestContainer) + + static GenericContainer mongoDBContainer + + + String getMongoDBHostName(){ + mongoDBContainer.getHost() + } + + String getMongoDBPort(){ + mongoDBContainer.getMappedPort(27017) + } + + def setupSpec() { + log.debug "Starting Redis test container" + mongoDBContainer = new GenericContainer<>( + DockerImageName.parse("mongo:7.0.12")) + .withExposedPorts(27017) + .waitingFor(Wait.forLogMessage(".*Waiting for connections.*\\n", 1)) + + mongoDBContainer.start() + log.debug "Started Redis test container" + } + + def cleanupSpec(){ + mongoDBContainer.stop() + } +} +