merged master

.github/scripts/delete-old-cloudsmith-artifacts.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+dnf install -y 'dnf-command(config-manager)' jq
+
+set -euo pipefail
+MAX_NUMBER_OF_RELEASES=40
+
+# Cloudsmith repo
+rpm --import 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/gpg.0F3DA3C70D35DA7B.key'
+curl -1sLf 'https://dl.cloudsmith.io/public/vespa/open-source-rpms/config.rpm.txt?distro=el&codename=8' > /tmp/vespa-open-source-rpms.repo
+dnf config-manager --add-repo '/tmp/vespa-open-source-rpms.repo'
+rm -f /tmp/vespa-open-source-rpms.repo
+
+# Allow the last Vespa 7 release to remain in the repo
+VERSIONS_TO_DELETE=$(dnf list -y --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-open-source-rpms vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | grep -v "7.594.36" | head -n -$MAX_NUMBER_OF_RELEASES)
+
+if [[ -z "$VERSIONS_TO_DELETE" ]]; then
+  echo "No old RPM versions to delete found. Exiting."
+  exit 0
+fi
+
+RPMS_TO_DELETE=$(mktemp)
+trap "rm -f $RPMS_TO_DELETE" EXIT
+
+for VERSION in $VERSIONS_TO_DELETE; do
+  curl -sLf --header 'accept: application/json' \
+    "https://api.cloudsmith.io/v1/packages/vespa/open-source-rpms/?query=version:${VERSION}" | jq -re '.[] | .slug' >> $RPMS_TO_DELETE
+done
+
+echo "Deleting the following RPMs:"
+cat $RPMS_TO_DELETE
+
+if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then
+    for RPMID in $(cat $RPMS_TO_DELETE); do
+      curl -sLf -X DELETE \
+        --header "X-Api-Key: $CLOUDSMITH_API_TOKEN" \
+        --header 'accept: application/json' \
+        "https://api.cloudsmith.io/v1/packages/vespa/open-source-rpms/$RPMID/"
+    done
+fi
+
+

.github/workflows/delete-old-versions-in-archive.yml

@@ -0,0 +1,38 @@
+name: Delete old Cloudsmith artifacts
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    paths:
+      - .github/workflows/delete-old-versions-in-archive.yml
+      - .github/scripts/delete-old-cloudsmith-artifacts.sh
+    branches:
+      - master
+
+  schedule:
+   - cron: '0 6 * * *'
+
+jobs:
+  delete-old-cloudsmith-artifacts:
+    runs-on: ubuntu-latest
+
+    container:
+        image: fedora:latest
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          CLOUDSMITH_API_TOKEN: ${{ secrets.CLOUDSMITH_API_TOKEN }}
+        volumes:
+          - ${{ github.workspace }}:/workspace
+
+    defaults:
+      run:
+        working-directory: /workspace
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Delete old artifacts
+        run: |
+          .github/scripts/delete-old-cloudsmith-artifacts.sh

.github/workflows/link-checker.yml

@@ -0,0 +1,28 @@
+name: Link checker
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: "0 2 * * 1-5"
+
+jobs:
+  test:
+    uses: vespa-engine/gh-actions/.github/workflows/jekyll-link-checker.yml@main
+    with:
+      ignore-urls: |-
+        /slack.vespa.ai/
+        /localhost:8080/
+        /127.0.0.1:3000/
+        /favicon.svg/
+        /main.jsx/
+      ignore-files: |-
+        /fnet/index.html/
+        /client/js/app/node_modules/
+      swap-urls: |-
+        (.*).md:\1.html

.github/workflows/verify-opensource-release-7days.yml

@@ -0,0 +1,61 @@
+name: Check OSS release age
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    paths:
+      - .github/workflows/verify-opensource-release-7days.yml
+    branches:
+      - master
+
+  schedule:
+   - cron: '0 0 * * *'
+
+jobs:
+  error-if-current-release-too-old:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Setup xq
+        run: |
+          sudo apt-get install -y xq
+
+      - name: Error if current release is too old
+        run: |
+          now_epoch=`date "+%s"`
+          echo "Now epoch: " $now_epoch
+
+          current_release_date=$(curl -sLf https://repo1.maven.org/maven2/com/yahoo/vespa/cloud-tenant-base/maven-metadata.xml | \
+          xq -x 'metadata/versioning/lastUpdated' | cut -c 1-8)
+          echo "Current release date: " $current_release_date
+          current_release_epoch=`date -d "$current_release_date" "+%s"`
+          echo "Current release epoch: " $current_release_epoch
+          release_age_days=$((($now_epoch-$current_release_epoch)/86400))
+          echo "Release age days: " $release_age_days
+
+          if [ "$release_age_days" -gt 7 ]; then
+            echo "Current open source release is older than 7 days"
+            exit 1
+          fi
+
+  error-if-docker-image-too-old:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Error if docker image is too old
+        run: |
+          now_epoch=`date "+%s"`
+          echo "Now epoch: " $now_epoch
+
+          image_date=$(curl -sLf https://hub.docker.com/v2/repositories/vespaengine/vespa/ | jq -re '.last_updated')
+          echo "Docker image last_updated: " $image_date
+          image_epoch=`date -d "$image_date" "+%s"`
+          echo "Docker image epoch: " $image_epoch
+          docker_image_age_days=$((($now_epoch-$image_epoch)/86400))
+          echo "Docker image age days: " $docker_image_age_days
+
+          if [ "$docker_image_age_days" -gt 7 ]; then
+            echo "Current Docker image is older than 7 days"
+            exit 1
+          fi
+
+

.github/workflows/verify-opensource-rpm-installable.yml

@@ -0,0 +1,30 @@
+name: Verify open source RPM installable
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    paths:
+      - .github/workflows/verify-opensource-rpm-installable.yml
+    branches:
+      - master
+
+  schedule:
+   - cron: '0 0 * * *'
+
+jobs:
+  verify-opensource-rpm-installable:
+    runs-on: ubuntu-latest
+
+    container:
+      image: almalinux:8
+
+    steps:
+      - name: Install Vespa
+        run: |
+          dnf list llvm-libs --showduplicates
+          dnf install -y dnf-plugins-core
+          dnf config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-8/group_vespa-vespa-epel-8.repo
+          dnf config-manager --enable powertools
+          dnf install -y epel-release
+          dnf install -y vespa

config-model-api/abi-spec.json

@@ -1788,12 +1788,14 @@
     ],
     "methods" : [
       "public void <init>(java.lang.String, java.lang.String, java.lang.String)",
+      "public void <init>(java.lang.String, java.lang.String, java.lang.String, java.lang.String)",
       "public void <init>(java.lang.String, java.lang.String, java.lang.String, java.util.Optional)",
       "public java.lang.String getName()",
       "public java.lang.String getAwsId()",
       "public java.lang.String getRole()",
       "public java.util.Optional getExternalId()",
       "public com.yahoo.config.model.api.TenantSecretStore withExternalId(java.lang.String)",
+      "public boolean isValid()",
       "public java.lang.String toString()",
       "public boolean equals(java.lang.Object)",
       "public int hashCode()"

config-model-api/src/main/java/com/yahoo/config/model/api/TenantSecretStore.java

@@ -5,6 +5,9 @@
 import java.util.Optional;
 
 /**
+ * TODO: this class can probably be moved to a non-PublicApi package
+ * TODO Vespa 9: remove
+ *
  * @author olaa
  */
 public class TenantSecretStore {
@@ -18,6 +21,10 @@ public TenantSecretStore(String name, String awsId, String role) {
         this(name, awsId, role, Optional.empty());
     }
 
+    public TenantSecretStore(String name, String awsId, String role, String externalId) {
+        this(name, awsId, role, Optional.of(externalId));
+    }
+
     public TenantSecretStore(String name, String awsId, String role, Optional<String> externalId) {
         this.name = name;
         this.awsId = awsId;
@@ -45,12 +52,19 @@ public TenantSecretStore withExternalId(String externalId) {
         return new TenantSecretStore(name, awsId, role, Optional.of(externalId));
     }
 
+    public boolean isValid() {
+        return !name.isBlank() &&
+                !awsId.isBlank() &&
+                !role.isBlank();
+    }
+
     @Override
     public String toString() {
         return "TenantSecretStore{" +
                 "name='" + name + '\'' +
                 ", awsId='" + awsId + '\'' +
                 ", role='" + role + '\'' +
+                ", externalId='" + (externalId.orElse("<EMPTY>") + '\'') +
                 '}';
     }
 
@@ -59,13 +73,12 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         TenantSecretStore that = (TenantSecretStore) o;
-        return name.equals(that.name) &&
-                awsId.equals(that.awsId) &&
-                role.equals(that.role);
+        return Objects.equals(name, that.name) && Objects.equals(awsId, that.awsId) && Objects.equals(role, that.role) && Objects.equals(externalId, that.externalId);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, awsId, role);
+        return Objects.hash(name, awsId, role, externalId);
     }
+
 }

config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java

@@ -42,25 +42,24 @@ public static SystemName defaultSystem() {
     }
 
     public static SystemName from(String value) {
-        switch (value.toLowerCase()) {
-            case "dev": return dev;
-            case "cd": return cd;
-            case "main": return main;
-            case "public": return Public;
-            case "publiccd": return PublicCd;
-            default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value));
-        }
+        return switch (value.toLowerCase()) {
+            case "dev" -> dev;
+            case "cd" -> cd;
+            case "main" -> main;
+            case "public" -> Public;
+            case "publiccd" -> PublicCd;
+            default -> throw new IllegalArgumentException(String.format("'%s' is not a valid system", value));
+        };
     }
 
     public String value() {
-        switch (this) {
-            case dev: return "dev";
-            case cd: return "cd";
-            case main: return "main";
-            case Public: return "public";
-            case PublicCd: return "publiccd";
-            default : throw new IllegalStateException();
-        }
+        return switch (this) {
+            case dev -> "dev";
+            case cd -> "cd";
+            case main -> "main";
+            case Public -> "public";
+            case PublicCd -> "publiccd";
+        };
     }
 
     /** Whether the system is similar to Public, e.g. PublicCd. */

config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java

@@ -99,7 +99,7 @@ protected JRTConnection pickNewConnectionRandomly(List<JRTConnection> sources) {
         return sources.get(ThreadLocalRandom.current().nextInt(0, sources.size()));
     }
 
-    protected List<JRTConnection> getSources() {
+    public List<JRTConnection> getSources() {
         List<JRTConnection> ret;
         synchronized (connections) {
             ret = new ArrayList<>(connections.values());

configdefinitions/src/vespa/proton.def

@@ -418,8 +418,8 @@ writefilter.memorylimit double default = 0.8
 ## put and update portion of feed is blocked.
 writefilter.disklimit double default = 0.8
 
-## Interval between sampling of disk and memory usage. Default is 10 seconds.
-writefilter.sampleinterval double default = 10.0
+## Interval between sampling of disk and memory usage. Default is 60 seconds.
+writefilter.sampleinterval double default = 60.0
 
 ## The size of the disk partition (in bytes) on which proton basedir is located.
 ## If set to 0, the disk size is sampled by looking at the filesystem space info.

configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java

@@ -574,15 +574,26 @@ private void serveFile(Request request) {
     private void setFileReferencesToDownload(Request req) {
         req.detach();
         rpcAuthorizer.authorizeFileRequest(req)
-                .thenRun(() -> { // okay to do in authorizer thread as downloadIfNeeded is async
+                .thenRun(() -> { // okay to do in authorizer thread as downloadFromSource is async
                     String[] fileReferenceStrings = req.parameters().get(0).asStringArray();
+
+                    // Download directly from the source that has the file reference, which
+                    // is the client that sent the request
+                    var client = req.target();
+                    var peerSpec = client.peerSpec();
                     Stream.of(fileReferenceStrings)
                             .map(FileReference::new)
-                            .forEach(fileReference -> downloader.downloadIfNeeded(
-                                    new FileReferenceDownload(fileReference,
-                                                              req.target().toString(),
-                                                              false /* downloadFromOtherSourceIfNotFound */)));
+                            .forEach(fileReference -> downloadFromSource(fileReference, client, peerSpec));
                     req.returnValues().add(new Int32Value(0));
                 });
     }
+
+    private void downloadFromSource(FileReference fileReference, Target client, Spec peerSpec) {
+        var fileReferenceDownload = new FileReferenceDownload(fileReference, client.toString());
+        var downloading = downloader.downloadFromSource(fileReferenceDownload, peerSpec);
+        log.log(FINE, () -> downloading
+                ? "Downloading file reference " + fileReference.value() + " from " + peerSpec
+                : "File reference " + fileReference.value() + " already exists");
+    }
+
 }