Merge branch 'master' into 223-improve-container-credentials-retrieva…

…l-using-workflow-compute-environment

.github/workflows/build.yml

@@ -72,6 +72,11 @@ jobs:
           AZURECR_PAT: ${{ secrets.AZURECR_PAT }}
           GOOGLECR_KEYS: ${{ secrets.GOOGLECR_KEYS }}
 
+      - name: Cleanup build workspace
+        if: always()
+        run: |
+          sudo rm -rf /home/runner/work/wave/wave/build-workspace
+
       - name: Release
         if: "contains(github.event.head_commit.message, '[release]')"
         run: |

VERSION

@@ -1 +1,2 @@
-1.6.2
+1.7.9
+

build.gradle

@@ -6,7 +6,7 @@ plugins {
     id 'io.seqera.wave.groovy-application-conventions'
     id "com.github.johnrengelman.shadow" version "7.1.1"
     id "io.micronaut.minimal.application" version "3.7.0"
-    id "com.google.cloud.tools.jib" version "3.4.0"
+    id "com.google.cloud.tools.jib" version "3.4.2"
     id 'org.asciidoctor.jvm.convert' version '3.3.2'
 }
 
@@ -34,7 +34,7 @@ dependencies {
     compileOnly("io.micronaut:micronaut-http-validation")
     implementation("jakarta.persistence:jakarta.persistence-api:3.0.0")
     api 'io.seqera:lib-mail:1.0.0'
-    api 'io.seqera:wave-api:0.9.1'
+    api 'io.seqera:wave-api:0.10.0'
     api 'io.seqera:wave-utils:0.12.0'
 
     implementation("io.micronaut:micronaut-http-client")

changelog.txt

@@ -1,4 +1,57 @@
 # Wave changelog
+1.7.9 - 29 May 2024
+- Improve JWT refresh handling  (#512) [751edb03]
+- Mark release as draft when ending with -Ax -Bx -RCx [ci skip] [66ba7ea7]
+- Removed unused imports (#510) [182e3f0d]
+- Fix Use atomic strategy for Cacheable ops [923ae67c]
+- Fix failing build [8adb52ab]
+
+1.7.8 - 15 May 2024
+- Prevent container freeze without custom repo (#499) [ec5932df]
+- Improve pairing record check (#500) [7ce86dc9]
+- Improve thread pool handling (#494) [e2c8c21b]
+
+1.7.7 - 11 May 2024 
+- Fix legacy tower endpoint validation (#495) [3257ad42]
+
+1.7.6 - 8 May 2024
+- Fix the handling of pip packages with Conda (#493) [35d9c1fd]
+- Fix Community repo name composition when namingStrategy is not specified [385c53cc]
+- Fix aggregate metrics issue (#487) [e4279602]
+
+1.7.5 - 2 May 2024
+- Fix community repo naming normalisation [a9b12d76]
+
+1.7.4 - 30 Apr 2024
+- Prevent use community registry w/o packages (2) [9eb11031]
+
+1.7.3 - 30 Apr 2024 
+- Prevent use community registry w/o packages [24a409de]
+
+1.7.2 - 29 Apr 2024 
+- Improve default container name handling [5325f2b1]
+- Improve error reporting [d6058919]
+- Prevent accessing to docker config [1c3d08d1]
+- Update metric service to get aggregated metrics  =(#472) [8ee9309f]
+- Update timeout max-rate to 20/2m [af0e4607]
+- Use IO executor consistently (#471) [ed57ce28]
+- Remove deprecated metrics v1alpha1 (#467) [3da8302d]
+
+1.7.1 - 23 Apr 2024
+- Tune thread pools [93e119ee] [0a5d08e7]
+- Added null check for container image name (#465) [30a6bdb6]
+- Move Redis tests config to setupSpec (#462) [d9fb2942]
+- Improve tests [9ab2cf42]
+
+1.7.0 - 22 Apr 2024
+- Improve build container naming strategy (#460) [31d459ed]
+- Helper methods refactor [54d2301b] [052acb32]
+- Refactor BuildRequest class (#456) [030ca310]
+- Improve request validation [6a3c4c79]
+- Bump trivy 0.50.1 [06ee665e]
+- Bump kaniko 1.22.0 [38a24afd]
+- Bump surrealdb 1.4.2 [5daf4ca7]
+
 1.6.1 - 11 Apr 2024
 - Undeprecate containerImage attribute in response object [23411028]
 
@@ -497,7 +550,7 @@ v0.25.8 - 22 Feb 2023
 
 v0.25.7 - 21 Feb 2023
 - Improve docker auth error reporting [eceffe85]
-- Fix log typo [ci skip] [030dbc67]
+- Fix log typo [030dbc67]
 
 v0.25.6 - 15 Feb 2023
 - Add landing redirection [ede9a242]
@@ -618,7 +671,7 @@ v0.22.0 - 23 Oct 2022
 - Improve the time format in the completion email (#169) [84357efe]
 - Make Tower user auth async (#166) [55f06451] <Jorge Aguilera>
 - Check if target image exist before building a new one (#173) [78a5fa2e]
-- Add basic k8s deployment [ci skip] [6c81171a]
+- Add basic k8s deployment [6c81171a]
 - Update ngrok launcher script [e72979ce]
 - Bump Micronaut 3.7.2 (#174) [28172705] 
 - Bump kaniko 1.9.1 (#175) [8bb72c47]

docs/cli/build-spack.mdx

@@ -4,7 +4,7 @@ title: Build a container from Spack packages
 
 The Wave CLI supports building a container from a list of [Spack] packages.
 
-:::warning
+:::caution
 Support for Spack packages is currently experimental.
 :::
 

docs/cli/install.mdx

@@ -5,12 +5,18 @@ title: Install the Wave CLI
 To install the `wave` CLI for your platform, complete the following steps:
 
 1.  Download the [latest version of the Wave CLI][download] for your platform.
+
 2.  In a new terminal, complete the following steps:
 
     1. Move the executable from your downloads folder to a location in your `PATH`, such as `~/bin`. For example: `mv wave-cli-0.8.0-macos-x86_64 ~/bin/wave`
     2. Ensure that the executable bit is set. For example: `chmod u+x ~/bin/wave`
 
-3.  Verify that you can build containers with Wave:
+3. You can also use [Homebrew](https://brew.sh/) in macos and linux, you can install like this:
+   ```bash
+   brew install seqeralabs/tap/wave-cli
+   ```
+
+4.  Verify that you can build containers with Wave:
 
     1.  Create a basic `Dockerfile`:
 

docs/guide.mdx

@@ -46,14 +46,33 @@ The Wave service implements API rate limits for API calls. Authenticated users h
 
 If an access token is provided, the following rate limits apply:
 
-- 100 container images per hour
-- 1,000 container images per minute
+- 100 container builds per hour
+- 1,000 container pulls per minute
 
-If an access token is not provided, the following rate limits apply:
+If an access token isn't provided, the following rate limits apply:
 
 - 25 container builds per day
 - 250 container pulls per hour
 
+## Known limitation
+
+### Use of sha256 digest in the image name
+
+The Wave does not support the use of sha256 digest in the image name, e.g. `ubuntu@sha256:3235...ce8f`, when using
+the augmentation process to extend container images.
+
+In order to reference a container via sha256 digest in the image name with Wave you will need to *freeze* image mode
+that will force the creation of a new container image using the container you have specified as base image.
+
+In your pipeline configuration, ensure that you specify the following settings:
+
+```groovy
+wave.enabled = true
+wave.freeze = true
+wave.strategy = ['dockerfile']
+wave.build.repository = 'docker.io/<user>/<repository>'
+```
+
 ## Tutorials
 
 ### Authenticate private repositories
@@ -89,7 +108,7 @@ wave.strategy = ['dockerfile','container']
 
 The above line instructs Wave to give the module `Dockerfile` priority over process `container` directives.
 
-:::warning
+:::caution
 Wave currently does not support `ADD`, `COPY` and other Dockerfile commands that access files in the host file system.
 :::
 

docs/index.mdx

@@ -22,9 +22,26 @@ Wave integrates with Seqera Platform credentials management enabling seamless ac
 Regulatory and security requirements sometimes dictate specific container images, but additional context is often needed.
 Wave enables any existing container to be extended without rebuilding it. Developers can add user-provided content such as custom scripts and logging agents, providing greater flexibility in the container’s configuration.
 
+Wave offers a flexible approach to container image management. It allows you to dynamically add custom layers to existing docker images, creating new images tailored to your specific needs.
+
+#### An example of Wave augmentation
+
+Imagine you have a base Ubuntu image in a container registry. Wave acts as a proxy between your docker client and the registry. When you request an augmented image, Wave intercepts the process.
+
+1. Base image layers download: The Docker client downloads the standard Ubuntu layers from the registry.
+2. Custom layer injection: Wave injects your custom layer, denoted by "ω", which could represent application code, libraries, configurations etc.
+3. New image creation: Wave combines the downloaded Ubuntu layers with your custom layer, effectively creating a new image on the fly.
+
+![](_images/wave_container_augmentation.png)
+
+#### Benefits of Wave augmentation
+
+1. Streamlined workflows: Wave simplifies your workflow by eliminating the need to manually build and manage custom images.
+2. Flexibility: You can easily modify the custom layer for different use cases, allowing for greater adaptability.
+
 ### Conda based containers
 
-Package management systems such as Conda and Bionconda simplify the installation of scientific software. However, there’s considerable friction when it comes to using those tools to deploy pipelines in cloud environments.
+Package management systems such as Conda and Bioconda simplify the installation of scientific software. However, there’s considerable friction when it comes to using those tools to deploy pipelines in cloud environments.
 Wave enables dynamic provisioning of container images from any Conda or Bioconda recipe. Just declare the Conda packages in your Nextflow pipeline and Wave will assemble the required container.
 
 ### Deploying containers across multi-clouds

docs/metrics.mdx

@@ -0,0 +1,56 @@
+---
+title: Wave usage metrics
+---
+
+Wave uses Redis to store its usage metrics for a specific date and/or a specific organization.
+
+These are stored using the following keys:
+
+- `pulls/d/YYYY-MM-DD`
+- `pulls/o/<org>`
+- `pulls/o/<org>/d/YYYY-MM-DD`
+- `fusion/d/YYYY-MM-DD`
+- `fusion/o/<org>`
+- `fusion/o/<org>/d/YYYY-MM-DD`
+- `builds/d/YYYY-MM-DD`
+- `builds/o/<org>`
+- `builds/o/<org>/d/YYYY-MM-DD`
+
+## Functionality
+
+### Store Builds
+
+When Wave launches a build, it also increments the values of following keys in Redis:
+
+- `builds/d/YYYY-MM-DD`
+- `builds/o/<org>`
+- `builds/o/<org>/d/YYYY-MM-DD`
+
+### Store Pulls
+
+Wave tracks the container image pulls using io.seqera.wave.filter.PullMetricsRequestsFilter, where it checks if `Content-Type` header contains one of the following values:
+
+- `application/vnd.docker.distribution.manifest.v2+json`
+- `application/vnd.oci.image.manifest.v1+json`
+- `application/vnd.docker.distribution.manifest.v1+prettyjws`
+- `application/vnd.docker.distribution.manifest.v1+json`
+
+Then it increments the values of following keys in Redis:
+
+- `pulls/d/YYYY-MM-DD`
+- `pulls/o/<org>`
+- `pulls/o/<org>/d/YYYY-MM-DD`
+
+Then, if the pulled container uses fusion, it increments the values of following keys in Redis:
+
+- `fusion/d/YYYY-MM-DD`
+- `fusion/o/<org>`
+- `fusion/o/<org>/d/YYYY-MM-DD`
+
+## How keys are created
+
+- When a request is made to wave, first it increments the key with current date. e.g. `builds/d/2024-04-23`.
+- Keys with organisation are only incremented if the user is authenticated means there is Seqera platform token in the request.
+- Wave extract the domain from the user email id (For example: `[email protected]`), which it gets from Seqera platform using the access token.
+- In this case, The organisation value will be `seqera.io`.
+- Then it increments the key with organisation. For example: `builds/o/seqera.io/d/2024-04-23` and `builds/o/seqera.io`.

docs/sidebar.json

@@ -21,6 +21,7 @@
     },
     "guide",
     "api",
+    "metrics",
     "faq"
   ]
 }

src/main/groovy/io/seqera/wave/WaveDefault.groovy

@@ -27,7 +27,6 @@ interface WaveDefault {
     final static public String DOCKER_IO = 'docker.io'
     final static public String DOCKER_REGISTRY_1 = 'https://registry-1.docker.io'
     final static public String DOCKER_INDEX_V1 = 'https://index.docker.io/v1/'
-    final static public String TOWER = 'tower'
 
     final static public Map<String,List<String>> ACCEPT_HEADERS = Map.of(
             'Accept', List.of(

src/main/groovy/io/seqera/wave/auth/RegistryCredentialsProviderImpl.groovy

@@ -68,9 +68,7 @@ class RegistryCredentialsProviderImpl implements RegistryCredentialsProvider {
 
     @Override
     RegistryCredentials getDefaultCredentials(ContainerPath container) {
-        return container && container.repository==buildConfig.defaultPublicRepository
-                ? getDefaultRepoCredentials0(container)
-                : getDefaultCredentials0(container?.registry)
+        return getDefaultCredentials0(container?.registry)
     }
 
     protected RegistryCredentials getDefaultCredentials0(String registry) {
@@ -115,8 +113,10 @@ class RegistryCredentialsProviderImpl implements RegistryCredentialsProvider {
             throw new IllegalArgumentException("Missing required parameter userId -- Unable to retrieve credentials for container repository '$container'")
 
         // use default credentials for default repositories
-        final repo = container.repository
-        if( repo==buildConfig.defaultBuildRepository || repo==buildConfig.defaultCacheRepository || repo==buildConfig.defaultPublicRepository)
+        // NOTE: this requires that 'defaultBuildRepository', 'defaultCacheRepository' and 'defaultPublicRepository' have a unique registry host name
+        // that means that for example docker.io/some/repo should not be used otherwise wave credentials could be used in place of user credentials
+        // for a repo having the same registry host
+        if( container.sameRegistry(buildConfig.defaultBuildRepository) || container.sameRegistry(buildConfig.defaultCacheRepository) || container.sameRegistry(buildConfig.defaultPublicRepository) )
             return getDefaultCredentials(container)
 
         return getUserCredentials0(container.registry, identity)

src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy

@@ -78,6 +78,9 @@ class BuildConfig {
     @Value('${wave.build.compress-caching:true}')
     Boolean compressCaching = true
 
+    @Value('${wave.build.reserved-words:[]}')
+    Set<String> reservedWords
+
     @PostConstruct
     private void init() {
         log.debug("Builder config: " +

src/main/groovy/io/seqera/wave/configuration/TokenConfig.groovy

@@ -41,6 +41,7 @@ interface TokenConfig {
         @Nullable
         Duration getDuration()
 
+        @Deprecated
         @Bindable(defaultValue = "10000")
         @Nullable
         int getMaxSize()