Merge pull request #356 from groldan/feature/pgsql-catalog-backend

New Catalog/Config backend for PostgreSQL 15+

.github/workflows/build.yaml

@@ -4,7 +4,7 @@ name: Build on any branch
 on:
   push:
     branches:
-      - '*'
+      - '**'
       - "!main"
     paths:
       - "Makefile"

docker-compose-pgconfig.yml

@@ -0,0 +1,72 @@
+version: "3.8"
+
+volumes:
+  pgconfig_data: # volume for postgresql data, used to store the geoserver config through pgsqlconfig backend
+
+#
+# Configures all geoserver services to use the postgresql database server with jdbcconfig as catalog backend.
+# Run with `docker-compose --compatibility -f docker-compose.yml -f docker-compose-jdbcconfig.yml up -d`
+#
+
+services:
+  pgconfig:
+    image: postgres:15
+    shm_size: 2g
+    environment:
+      POSTGRES_DB: pgconfig
+      POSTGRES_USER: pgconfig
+      POSTGRES_PASSWORD: pgconfig
+    ports:
+      - 54322:5432
+    networks:
+      - gs-cloud-network
+    volumes:
+      - pgconfig_data:/var/lib/postgresql/data
+    deploy:
+      resources:
+        limits:
+          cpus: '4.0'
+          memory: 4G
+
+
+  wfs:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig
+
+  wms:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig
+
+  wcs:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig
+
+  rest:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig
+
+  webui:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig
+
+  gwc:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig
+
+  wps:
+    environment:
+      SPRING_PROFILES_ACTIVE: "${DEFAULT_PROFILES},pgconfig"
+    depends_on:
+      - pgconfig

pom.xml

@@ -136,6 +136,11 @@
         <artifactId>gs-cloud-catalog-backend-catalog-service</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.geoserver.cloud.catalog.backend</groupId>
+        <artifactId>gs-cloud-catalog-backend-pgsql</artifactId>
+        <version>${project.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.geoserver.cloud</groupId>
         <artifactId>gwc-cloud-core</artifactId>

src/apps/geoserver/gwc/src/main/resources/bootstrap.yml

@@ -61,6 +61,7 @@ spring:
       - org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration
       - org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
       - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
 
 # override default of true, this service does not use the registry (when eureka client is enabled)
 eureka.client.fetch-registry: false

src/apps/geoserver/restconfig/src/main/resources/bootstrap.yml

@@ -35,6 +35,7 @@ spring:
       - org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
       - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
       - org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
 
 
 # override default of true, this service does not use the registry (when eureka client is enabled)

src/apps/geoserver/wcs/src/main/resources/bootstrap.yml

@@ -34,6 +34,7 @@ spring:
       - org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration
       - org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
       - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
 
 # override default of true, this service does not use the registry (when eureka client is enabled)
 eureka.client.fetch-registry: false

src/apps/geoserver/webui/src/main/resources/bootstrap.yml

@@ -42,6 +42,7 @@ spring:
       - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
       - org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration
       - org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
       # Force disabling GWC UI, it's embedded in gwc-core, so can't have a ConditionalOnClass
       - org.geoserver.cloud.autoconfigure.web.gwc.GeoWebCacheUIAutoConfiguration
 

src/apps/geoserver/wfs/src/main/resources/bootstrap.yml

@@ -36,6 +36,7 @@ spring:
       - org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration
       - org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
       - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
 
 # override default of true, this service does not use the registry (when eureka client is enabled)
 eureka.client.fetch-registry: false

src/apps/geoserver/wms/src/main/resources/bootstrap.yml

@@ -43,6 +43,7 @@ spring:
       - org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration
       # Force disabling GWC UI, it's embedded in gwc-core, so can't have a ConditionalOnClass
       - org.geoserver.cloud.autoconfigure.web.gwc.GeoWebCacheUIAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
 
 # override default of true, this service does not use the registry (when eureka client is enabled)
 eureka.client.fetch-registry: false

src/apps/geoserver/wps/src/main/resources/bootstrap.yml

@@ -34,6 +34,7 @@ spring:
       - org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration
       - org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration
       - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
+      - org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
 
 # override default of true, this service does not use the registry (when eureka client is enabled)
 eureka.client.fetch-registry: false

src/catalog/backends/common/src/main/java/org/geoserver/cloud/autoconfigure/catalog/backend/core/DefaultUpdateSequenceAutoConfiguration.java

@@ -4,6 +4,7 @@
  */
 package org.geoserver.cloud.autoconfigure.catalog.backend.core;
 
+import org.geoserver.config.GeoServer;
 import org.geoserver.platform.config.DefaultUpdateSequence;
 import org.geoserver.platform.config.UpdateSequence;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
@@ -14,7 +15,7 @@
 @ConditionalOnMissingBean(UpdateSequence.class)
 public class DefaultUpdateSequenceAutoConfiguration {
     @Bean
-    UpdateSequence defaultUpdateSequence() {
-        return new DefaultUpdateSequence();
+    UpdateSequence defaultUpdateSequence(GeoServer gs) {
+        return new DefaultUpdateSequence(gs);
     }
 }

src/catalog/backends/datadir/src/test/java/org/geoserver/cloud/autoconfigure/catalog/backend/datadir/DataDirectoryUpdateSequenceTest.java

@@ -4,17 +4,15 @@
  */
 package org.geoserver.cloud.autoconfigure.catalog.backend.datadir;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
 import org.geoserver.cloud.config.catalog.backend.datadirectory.DataDirectoryBackendConfiguration;
 import org.geoserver.cloud.config.catalog.backend.datadirectory.DataDirectoryUpdateSequence;
-import org.junit.jupiter.api.Test;
+import org.geoserver.config.GeoServer;
+import org.geoserver.platform.config.UpdateSequence;
+import org.geoserver.platform.config.UpdateSequenceConformanceTest;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 
-import java.util.stream.IntStream;
-
 /**
  * Test {@link DataDirectoryBackendConfiguration} through {@link DataDirectoryAutoConfiguration}
  * when {@code geoserver.backend.data-directory.enabled=true}
@@ -26,32 +24,18 @@
             "geoserver.backend.dataDirectory.location=/tmp/data_dir_autoconfiguration_test"
         })
 @ActiveProfiles("test")
-public class DataDirectoryUpdateSequenceTest {
+public class DataDirectoryUpdateSequenceTest implements UpdateSequenceConformanceTest {
 
     private @Autowired DataDirectoryUpdateSequence updateSequence;
+    private @Autowired GeoServer geoserver;
 
-    public @Test void sequentialTest() {
-        final long initial = updateSequence.currValue();
-        long v = updateSequence.currValue();
-        assertEquals(initial, v);
-        v = updateSequence.nextValue();
-        assertEquals(1 + initial, v);
-        v = updateSequence.currValue();
-        assertEquals(1 + initial, v);
-        v = updateSequence.nextValue();
-        assertEquals(2 + initial, v);
-        v = updateSequence.currValue();
-        assertEquals(2 + initial, v);
+    @Override
+    public UpdateSequence getUpdataSequence() {
+        return updateSequence;
     }
 
-    public @Test void multiThreadedTest() {
-        final int incrementCount = 10_000;
-        final long initial = updateSequence.currValue();
-        final long expected = initial + incrementCount;
-
-        IntStream.range(0, incrementCount).parallel().forEach(i -> updateSequence.nextValue());
-
-        long v = updateSequence.currValue();
-        assertEquals(expected, v);
+    @Override
+    public GeoServer getGeoSever() {
+        return geoserver;
     }
 }

src/catalog/backends/jdbcconfig/pom.xml

@@ -83,5 +83,12 @@
       <version>1.4.200</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.geoserver.cloud.catalog</groupId>
+      <artifactId>gs-cloud-catalog-plugin</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

src/catalog/backends/jdbcconfig/src/main/java/org/geoserver/cloud/config/catalog/backend/jdbcconfig/JDBCConfigBackendConfigurer.java

@@ -136,7 +136,9 @@ public JDBCConfigBackendConfigurer(
     public @Override UpdateSequence updateSequence() {
         DataSource dataSource = jdbcConfigDataSource();
         CloudJdbcConfigProperties props = jdbcConfigProperties();
-        return new JdbcConfigUpdateSequence(dataSource, props);
+        GeoServerFacade geoserverFacade = geoserverFacade();
+        ConfigDatabase db = jdbcConfigDB();
+        return new JdbcConfigUpdateSequence(dataSource, props, geoserverFacade, db);
     }
 
     @Bean

src/catalog/backends/jdbcconfig/src/main/java/org/geoserver/cloud/config/catalog/backend/jdbcconfig/JdbcConfigUpdateSequence.java

@@ -9,6 +9,9 @@
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 
+import org.geoserver.config.GeoServerFacade;
+import org.geoserver.config.GeoServerInfo;
+import org.geoserver.jdbcconfig.internal.ConfigDatabase;
 import org.geoserver.platform.config.UpdateSequence;
 import org.springframework.beans.factory.InitializingBean;
 
@@ -29,6 +32,8 @@ public class JdbcConfigUpdateSequence implements UpdateSequence, InitializingBea
 
     private final @NonNull DataSource dataSource;
     private final @NonNull CloudJdbcConfigProperties props;
+    private final @NonNull GeoServerFacade geoServer;
+    private final @NonNull ConfigDatabase db;
 
     private String incrementAndGetQuery;
     private String getQuery;
@@ -39,8 +44,14 @@ public long currValue() {
     }
 
     @Override
-    public long nextValue() {
-        return runAndGetLong(this.incrementAndGetQuery);
+    public synchronized long nextValue() {
+        long nextValue = runAndGetLong(this.incrementAndGetQuery);
+        GeoServerInfo global = geoServer.getGlobal();
+        if (global != null) {
+            global.setUpdateSequence(nextValue);
+            db.save(global);
+        }
+        return nextValue;
     }
 
     @Override

src/catalog/backends/jdbcconfig/src/test/java/org/geoserver/cloud/autoconfigure/catalog/backend/jdbcconfig/JdbcConfigUpdateSequenceTest.java

@@ -4,47 +4,36 @@
  */
 package org.geoserver.cloud.autoconfigure.catalog.backend.jdbcconfig;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
 import org.geoserver.cloud.config.catalog.backend.jdbcconfig.JdbcConfigUpdateSequence;
+import org.geoserver.config.GeoServer;
+import org.geoserver.platform.config.UpdateSequence;
+import org.geoserver.platform.config.UpdateSequenceConformanceTest;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 
-import java.util.stream.IntStream;
-
 @SpringBootTest(
         classes = AutoConfigurationTestConfiguration.class,
         properties = "geoserver.backend.jdbcconfig.enabled=true")
-public class JdbcConfigUpdateSequenceTest extends JDBCConfigTest {
+public class JdbcConfigUpdateSequenceTest extends JDBCConfigTest
+        implements UpdateSequenceConformanceTest {
 
     private @Autowired JdbcConfigUpdateSequence updateSequence;
 
-    @Disabled(
-            "Couldn't get rid of the DB closed error if running more than one test, so better run the parallel one")
-    public @Test void testUpdateSequence() {
-        final long initial = updateSequence.currValue();
-        long v = updateSequence.currValue();
-        assertEquals(initial, v);
-        v = updateSequence.nextValue();
-        assertEquals(1 + initial, v);
-        v = updateSequence.currValue();
-        assertEquals(1 + initial, v);
-        v = updateSequence.nextValue();
-        assertEquals(2 + initial, v);
-        v = updateSequence.currValue();
-        assertEquals(2 + initial, v);
+    @Override
+    public UpdateSequence getUpdataSequence() {
+        return updateSequence;
     }
 
-    public @Test void multiThreadedTest() {
-        final int incrementCount = 10_000;
-        final long initial = updateSequence.currValue();
-        final long expected = initial + incrementCount;
-
-        IntStream.range(0, incrementCount).parallel().forEach(i -> updateSequence.nextValue());
+    @Override
+    public GeoServer getGeoSever() {
+        return super.geoServer;
+    }
 
-        long v = updateSequence.currValue();
-        assertEquals(expected, v);
+    @Disabled(
+            "Couldn't get rid of the DB closed error if running more than one test, so better just run the parallel one")
+    public @Override @Test void testUpdateSequence() {
+        // no-op
     }
 }

src/catalog/backends/jdbcconfig/src/test/resources/application.yml

@@ -5,19 +5,14 @@ spring:
     allow-bean-definition-overriding: true
     # false by default since spring-boot 2.6.0, breaks geoserver initialization
     allow-circular-references: true
-  cloud.bus.enabled: false
   autoconfigure:
     exclude:
     - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
 
-
-eureka.client.enabled: false
-
 geoserver:
   backend:
     jdbcconfig:
       enabled: false
-      web.enabled: false
       initdb: true
       cache-directory: ${java.io.tmpdir}/geoserver-jdbcconfig-cache
       datasource: