Automatic cert renewal policy and self-signed CA creation (#20842)

* Automatic cert renewal policy and self-signed CA creation * Datanode CSR rate limited, added IT * selfsigned_startup property name for automatic cert renewal and selfsigned CA configuration * Add warning to the insecure configuration * default for relaxedHTTPSValidation in RestOperationParameters * add changelog * code cleanup * Simplify rate limiting in DataNodeCertRenewalPeriodical --------- Co-authored-by: Matthias Oesterheld <[email protected]>
Graylog2 · Nov 14, 2024 · 671eacc · 671eacc
1 parent 60e1ca3
commit 671eacc
Show file tree

Hide file tree

Showing 15 changed files with 353 additions and 22 deletions.
diff --git a/changelog/unreleased/pr-20842.toml b/changelog/unreleased/pr-20842.toml
@@ -0,0 +1,5 @@
+type = "c"
+message = "Replace datanode insecure_startup configuration with selfsigned_startup, providing full selfsigned SSL setup"
+
+issues = ["18911"]
+pulls = ["20842"]
diff --git a/...src/main/java/org/graylog/datanode/bootstrap/preflight/DataNodeCertRenewalPeriodical.java b/...src/main/java/org/graylog/datanode/bootstrap/preflight/DataNodeCertRenewalPeriodical.java
@@ -37,7 +37,8 @@
 @Singleton
 public class DataNodeCertRenewalPeriodical extends Periodical {
     private static final Logger LOG = LoggerFactory.getLogger(DataNodeCertRenewalPeriodical.class);
-    public static final Duration PERIODICAL_DURATION = Duration.ofMinutes(30);
+    public static final Duration PERIODICAL_DURATION = Duration.ofSeconds(2);
+    public static final Duration CSR_TRIGGER_PERIOD_LIMIT = Duration.ofMinutes(5);
 
     private final DatanodeKeystore datanodeKeystore;
     private final Supplier<RenewalPolicy> renewalPolicySupplier;
@@ -46,23 +47,20 @@ public class DataNodeCertRenewalPeriodical extends Periodical {
 
     private final Supplier<Boolean> isServerInPreflightMode;
 
+    private Instant lastCsrRequest;
+
     @Inject
     public DataNodeCertRenewalPeriodical(DatanodeKeystore datanodeKeystore, ClusterConfigService clusterConfigService, CsrRequester csrRequester, PreflightConfigService preflightConfigService) {
         this(datanodeKeystore, () -> clusterConfigService.get(RenewalPolicy.class), csrRequester, () -> isInPreflight(preflightConfigService));
     }
 
-    private static boolean isInPreflight(PreflightConfigService preflightConfigService) {
-        return preflightConfigService.getPreflightConfigResult() != PreflightConfigResult.FINISHED;
-    }
-
     protected DataNodeCertRenewalPeriodical(DatanodeKeystore datanodeKeystore, Supplier<RenewalPolicy> renewalPolicySupplier, CsrRequester csrRequester, Supplier<Boolean> isServerInPreflightMode) {
         this.datanodeKeystore = datanodeKeystore;
         this.renewalPolicySupplier = renewalPolicySupplier;
         this.csrRequester = csrRequester;
         this.isServerInPreflightMode = isServerInPreflightMode;
     }
 
-
     @Override
     public void doRun() {
         if (isServerInPreflightMode.get()) {
@@ -80,15 +78,22 @@ public void doRun() {
                         case MANUAL -> manualRenewal();
                     }
                 });
+    }
 
+    private static boolean isInPreflight(PreflightConfigService preflightConfigService) {
+        return preflightConfigService.getPreflightConfigResult() != PreflightConfigResult.FINISHED;
     }
 
     private void manualRenewal() {
         LOG.debug("Manual renewal, ignoring on the datanode side for now");
     }
 
     private void automaticRenewal() {
-        csrRequester.triggerCertificateSigningRequest();
+        final Instant now = Instant.now();
+        if (lastCsrRequest == null || now.minus(CSR_TRIGGER_PERIOD_LIMIT).isAfter(lastCsrRequest)) {
+            lastCsrRequest = now;
+            csrRequester.triggerCertificateSigningRequest();
+        }
     }
 
     private boolean needsNewCertificate(RenewalPolicy renewalPolicy) {

diff --git a/...node/src/main/java/org/graylog/datanode/configuration/variants/InSecureConfiguration.java b/...node/src/main/java/org/graylog/datanode/configuration/variants/InSecureConfiguration.java
@@ -18,16 +18,22 @@
 
 import org.graylog.datanode.Configuration;
 import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+@Deprecated
 public class InSecureConfiguration implements SecurityConfigurationVariant {
 
+    private static final Logger LOG = LoggerFactory.getLogger(InSecureConfiguration.class);
+
     @Override
     public boolean isConfigured(final Configuration localConfiguration) {
         return localConfiguration.isInsecureStartup();
     }
 
     @Override
     public OpensearchSecurityConfiguration build() throws KeyStoreStorageException {
+        LOG.warn("Insecure configuration is deprecated. Please use selfsigned_setup to create fully encrypted setups.");
         return OpensearchSecurityConfiguration.disabled();
     }
 }
diff --git a/full-backend-tests/src/test/java/org/graylog/datanode/DatanodeSelfsignedStartupIT.java b/full-backend-tests/src/test/java/org/graylog/datanode/DatanodeSelfsignedStartupIT.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode;
+
+import com.github.joschi.jadconfig.util.Duration;
+import com.github.rholder.retry.RetryException;
+import io.restassured.response.ValidatableResponse;
+import org.graylog.testing.completebackend.ContainerizedGraylogBackend;
+import org.graylog.testing.completebackend.Lifecycle;
+import org.graylog.testing.completebackend.apis.GraylogApis;
+import org.graylog.testing.containermatrix.SearchServer;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration;
+import org.graylog.testing.restoperations.DatanodeOpensearchWait;
+import org.graylog.testing.restoperations.RestOperationParameters;
+import org.graylog2.security.IndexerJwtAuthTokenProvider;
+import org.graylog2.security.JwtSecret;
+import org.hamcrest.Matchers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.concurrent.ExecutionException;
+
+@ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS, searchVersions = SearchServer.DATANODE_DEV,
+                                   additionalConfigurationParameters = {
+                                           @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = "GRAYLOG_DATANODE_INSECURE_STARTUP", value = "false"),
+                                           @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = "GRAYLOG_SELFSIGNED_STARTUP", value = "true"),
+                                           @ContainerMatrixTestsConfiguration.ConfigurationParameter(key = "GRAYLOG_ELASTICSEARCH_HOSTS", value = ""),
+                                   })
+public class DatanodeSelfsignedStartupIT {
+
+
+    private final Logger log = LoggerFactory.getLogger(DatanodeProvisioningIT.class);
+
+    private final GraylogApis apis;
+
+    public DatanodeSelfsignedStartupIT(GraylogApis apis) {
+        this.apis = apis;
+    }
+
+    @ContainerMatrixTest
+    public void testSelfsignedStartup() throws ExecutionException, RetryException {
+        testEncryptedConnectionToOpensearch();
+    }
+
+
+    private int getOpensearchPort() {
+        final String indexerHostAddress = apis.backend().searchServerInstance().getHttpHostAddress();
+        return Integer.parseInt(indexerHostAddress.split(":")[1]);
+    }
+
+    private void testEncryptedConnectionToOpensearch() throws ExecutionException, RetryException {
+        try {
+            final ValidatableResponse response = new DatanodeOpensearchWait(RestOperationParameters.builder()
+                    .port(getOpensearchPort())
+                    .relaxedHTTPSValidation(true)
+                    .jwtTokenProvider(new IndexerJwtAuthTokenProvider(new JwtSecret(ContainerizedGraylogBackend.PASSWORD_SECRET), Duration.seconds(120), Duration.seconds(60)))
+                    .build())
+                    .waitForNodesCount(1);
+
+            response.assertThat().body("status", Matchers.equalTo("green"));
+        } catch (Exception e) {
+            log.error("Could not connect to Opensearch\n" + apis.backend().getSearchLogs());
+            throw e;
+        }
+    }
+}
diff --git a/graylog2-server/src/main/java/org/graylog2/Configuration.java b/graylog2-server/src/main/java/org/graylog2/Configuration.java
@@ -37,6 +37,7 @@
 import org.graylog2.cluster.leader.LeaderElectionMode;
 import org.graylog2.cluster.leader.LeaderElectionService;
 import org.graylog2.cluster.lock.MongoLockService;
+import org.graylog2.configuration.Documentation;
 import org.graylog2.configuration.converters.JavaDurationConverter;
 import org.graylog2.notifications.Notification;
 import org.graylog2.outputs.BatchSizeConfig;
@@ -249,6 +250,14 @@ public class Configuration extends CaConfiguration {
     @Parameter(value = "field_value_suggestion_mode", required = true, converter = FieldValueSuggestionModeConverter.class)
     private FieldValueSuggestionMode fieldValueSuggestionMode = FieldValueSuggestionMode.ON;
 
+    @Documentation("""
+            Enabling this parameter will activate automatic security configuration. Graylog server will
+            set a default 30-day automatic certificate renewal policy and create a self-signed CA. This CA
+            will be used to sign certificates for SSL communication between the server and datanodes.
+            """)
+    @Parameter(value = "selfsigned_startup")
+    private boolean selfsignedStartup = false;
+
     public static final String INSTALL_HTTP_CONNECTION_TIMEOUT = "install_http_connection_timeout";
     public static final String INSTALL_OUTPUT_BUFFER_DRAINING_INTERVAL = "install_output_buffer_drain_interval";
     public static final String INSTALL_OUTPUT_BUFFER_DRAINING_MAX_RETRIES = "install_output_buffer_max_retries";
@@ -559,6 +568,10 @@ public int getQueryLatencyMonitoringWindowSize() {
         return queryLatencyMonitoringWindowSize;
     }
 
+    public boolean selfsignedStartupEnabled() {
+        return selfsignedStartup;
+    }
+
     public static class NodeIdFileValidator implements Validator<String> {
         @Override
         public void validate(String name, String path) throws ValidationException {

diff --git a/...erver/src/main/java/org/graylog2/cluster/preflight/GraylogServerProvisioningBindings.java b/...erver/src/main/java/org/graylog2/cluster/preflight/GraylogServerProvisioningBindings.java
@@ -21,12 +21,14 @@
 import org.graylog2.bootstrap.preflight.GraylogCertificateProvisionerImpl;
 import org.graylog2.cluster.certificates.CertificateExchange;
 import org.graylog2.cluster.certificates.CertificateExchangeImpl;
+import org.graylog2.configuration.IndexerDiscoverySecurityAutoconfig;
 
 public class GraylogServerProvisioningBindings extends AbstractModule {
 
     @Override
     protected void configure() {
         bind(CertificateExchange.class).to(CertificateExchangeImpl.class);
         bind(GraylogCertificateProvisioner.class).to(GraylogCertificateProvisionerImpl.class);
+        bind(IndexerDiscoverySecurityAutoconfig.class);
     }
 }
diff --git a/...og2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryCertProvisioning.java b/...og2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryCertProvisioning.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.configuration;
+
+import jakarta.inject.Inject;
+import org.graylog2.bootstrap.preflight.GraylogCertificateProvisioner;
+
+public class IndexerDiscoveryCertProvisioning implements IndexerDiscoveryListener {
+
+    private final GraylogCertificateProvisioner graylogCertificateProvisioner;
+
+    @Inject
+    public IndexerDiscoveryCertProvisioning(GraylogCertificateProvisioner graylogCertificateProvisioner) {
+        this.graylogCertificateProvisioner = graylogCertificateProvisioner;
+    }
+
+    @Override
+    public void beforeIndexerDiscovery() {
+
+    }
+
+    @Override
+    public void onDiscoveryRetry() {
+        // let's try to provision certificates, maybe there are datanodes waiting for these
+        graylogCertificateProvisioner.runProvisioning();
+    }
+}
diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryListener.java b/graylog2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryListener.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.configuration;
+
+public interface IndexerDiscoveryListener {
+    /**
+     * Triggered before we start with indexer discovery. Won't be triggered if there are any indexers
+     * explicitly defined in the configuration.
+     */
+    void beforeIndexerDiscovery();
+
+    /**
+     * Triggered after each unsuccessful retry during indexer discovery
+     */
+    void onDiscoveryRetry();
+}
diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryModule.java b/graylog2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryModule.java
@@ -18,6 +18,7 @@
 
 import com.google.inject.AbstractModule;
 import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
 import org.graylog2.bindings.providers.MongoConnectionProvider;
 import org.graylog2.bootstrap.preflight.PreflightConfigService;
 import org.graylog2.bootstrap.preflight.PreflightConfigServiceImpl;
@@ -34,11 +35,22 @@
 public class IndexerDiscoveryModule extends AbstractModule {
     @Override
     protected void configure() {
+        registerIndexerDiscoveryListener(IndexerDiscoveryCertProvisioning.class);
+        registerIndexerDiscoveryListener(IndexerDiscoverySecurityAutoconfig.class);
+
         bind(new TypeLiteral<List<URI>>() {}).annotatedWith(IndexerHosts.class).toProvider(IndexerDiscoveryProvider.class).asEagerSingleton();
         bind(Boolean.class).annotatedWith(RunsWithDataNode.class).toProvider(RunsWithDataNodeDiscoveryProvider.class).asEagerSingleton();
         bind(new TypeLiteral<NodeService<DataNodeDto>>() {}).to(DataNodeClusterService.class);
         bind(PreflightConfigService.class).to(PreflightConfigServiceImpl.class);
         bind(MongoConnection.class).toProvider(MongoConnectionProvider.class);
         bind(JwtSecret.class).toProvider(JwtSecretProvider.class).asEagerSingleton();
     }
+
+    protected void registerIndexerDiscoveryListener(Class<? extends IndexerDiscoveryListener> listener) {
+        indexerDiscoveryListerers().addBinding().to(listener);
+    }
+
+    protected Multibinder<IndexerDiscoveryListener> indexerDiscoveryListerers() {
+        return Multibinder.newSetBinder(binder(), IndexerDiscoveryListener.class);
+    }
 }
diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryProvider.java b/graylog2-server/src/main/java/org/graylog2/configuration/IndexerDiscoveryProvider.java
@@ -27,7 +27,6 @@
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 import jakarta.inject.Provider;
-import org.graylog2.bootstrap.preflight.GraylogCertificateProvisioner;
 import org.graylog2.bootstrap.preflight.PreflightConfigResult;
 import org.graylog2.bootstrap.preflight.PreflightConfigService;
 import org.graylog2.cluster.Node;
@@ -39,6 +38,7 @@
 import java.net.URI;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -53,7 +53,8 @@ public class IndexerDiscoveryProvider implements Provider<List<URI>> {
     private final List<URI> hosts;
     private final PreflightConfigService preflightConfigService;
     private final NodeService<DataNodeDto> nodeService;
-    private final GraylogCertificateProvisioner graylogCertificateProvisioner;
+
+    private final Set<IndexerDiscoveryListener> indexerDiscoveryListeners;
 
     private final Supplier<List<URI>> resultsCachingSupplier;
 
@@ -68,16 +69,17 @@ public IndexerDiscoveryProvider(
             @Named("datanode_startup_connection_delay") Duration delayBetweenAttempts,
             PreflightConfigService preflightConfigService,
             NodeService<DataNodeDto> nodeService,
-            GraylogCertificateProvisioner graylogCertificateProvisioner) {
+            Set<IndexerDiscoveryListener> indexerDiscoveryListeners) {
         this.hosts = hosts;
         this.connectionAttempts = connectionAttempts;
         this.delayBetweenAttempts = delayBetweenAttempts;
         this.preflightConfigService = preflightConfigService;
         this.nodeService = nodeService;
-        this.graylogCertificateProvisioner = graylogCertificateProvisioner;
+        this.indexerDiscoveryListeners = indexerDiscoveryListeners;
         this.resultsCachingSupplier = Suppliers.memoize(this::doGet);
     }
 
+
     @Override
     public List<URI> get() {
         return resultsCachingSupplier.get();
@@ -90,6 +92,8 @@ private List<URI> doGet() {
             return hosts;
         }
 
+        indexerDiscoveryListeners.forEach(IndexerDiscoveryListener::beforeIndexerDiscovery);
+
         final PreflightConfigResult preflightResult = preflightConfigService.getPreflightConfigResult();
 
         // if preflight is finished, we assume that there will be some datanode registered via node-service.
@@ -109,8 +113,7 @@ public void onRetry(Attempt attempt) {
                                     }
 
                                 }
-                                // let's try to provision certificates, maybe there are datanodes waiting for these
-                                graylogCertificateProvisioner.runProvisioning();
+                                indexerDiscoveryListeners.forEach(IndexerDiscoveryListener::onDiscoveryRetry);
                             }
                         })
                         .withWaitStrategy(WaitStrategies.fixedWait(delayBetweenAttempts.getQuantity(), delayBetweenAttempts.getUnit()))