Merge pull request #39 from groldan/GSMEL-334_read_all_postgis_schemas

Add ability to publish data belonging to all DB schemas

README.md

@@ -9,7 +9,6 @@ Those components are created with the following requirements:
 * Maven 3.8+
 * Docker
 
-
 ### Building
 
 Compile, test, install:
@@ -40,13 +39,19 @@ SPRING_PROFILES_ACTIVE: postgis
 POSTGRES_HOST: postgis
 POSTGRES_PORT: 5432
 POSTGRES_DB: postgis
-POSTGRES_SCHEMA: opendataindex
 POSTGRES_USER: postgis
 POSTGRES_PASSWORD: postgis
 POSTGRES_POOL_MAXSIZE: 20
 POSTGRES_POOL_MINSIZE: 0
 ```
 
+### Serving data from multiple PostgreSQL schemas
+
+The above configuration will result in the `ogc-features` application serving data from all PostgreSQL schemas,
+and prefixing the collection names with the schema names.
+
+Checkout the [postgis.md](postgis.md) file for detailed instructions on how to configure schema inclusion and aliasing.
+
 ### Development
 
 See the [OGC Features API](src/services/ogc-features/README.md) README for more information.

compose-entrypoint.d/gdal-import-sample-data.sh

@@ -1,25 +1,16 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e
 
-# Load sample data PostGIS into $POSTGRES_DB
-# for csv in `ls /sample-data/*csv`
-# do
-# 	echo "Importing sample dataset $csv into database $POSTGRES_DB as user $POSTGRES_USER"
-# 	ogr2ogr -f PostgreSQL PG:"host='localhost' active_schema='opendataindex' user='$POSTGRES_USER' dbname='$POSTGRES_DB' password='$POSTGRES_PASSWORD'" \
-# 	$csv -oo AUTODETECT_TYPE=YES \
-# 	-oo X_POSSIBLE_NAMES=Longitude,LON \
-# 	-oo Y_POSSIBLE_NAMES=Latitude,LAT \
-# 	-oo KEEP_GEOM_COLUMNS=NO
-# done
-
-#sleep 2 
-echo "Importing sample datasets sample-datasets.gpkg into database $POSTGRES_HOST:$POSTGRES_DB as user $POSTGRES_USER"
-export SQLITE_LIST_ALL_TABLES=YES
-ogr2ogr -f PostgreSQL \
--lco LAUNDER=NO \
-PG:"host='$POSTGRES_HOST' active_schema='$POSTGRES_SCHEMA' user='$POSTGRES_USER' dbname='$POSTGRES_DB' password='$POSTGRES_PASSWORD'" \
--oo LIST_ALL_TABLES=YES \
--lco OVERWRITE=YES \
-/sample-data/sample-datasets.gpkg
+for POSTGRES_SCHEMA in "opendataindex" "dsp_esterra" "dsp_ileo" "ville_Armentières" "ville_Villeneuve-d\'Ascq"
+do
+	echo "Importing sample datasets sample-datasets.gpkg into $POSTGRES_HOST:$POSTGRES_DB, schema \"$POSTGRES_SCHEMA\", as user $POSTGRES_USER"
+	export SQLITE_LIST_ALL_TABLES=YES
+	ogr2ogr -f PostgreSQL \
+	-lco LAUNDER=NO \
+	PG:"host='$POSTGRES_HOST' active_schema='$POSTGRES_SCHEMA' user='$POSTGRES_USER' dbname='$POSTGRES_DB' password='$POSTGRES_PASSWORD'" \
+	-oo LIST_ALL_TABLES=YES \
+	-lco OVERWRITE=YES \
+	/sample-data/sample-datasets.gpkg
+done
 

compose-entrypoint.d/postgis-create-opendataindex-schema.sql

@@ -1,2 +1,7 @@
 CREATE SCHEMA opendataindex;
 SET search_path to opendataindex;
+
+CREATE SCHEMA dsp_esterra;
+CREATE SCHEMA dsp_ileo;
+CREATE SCHEMA "ville_Armentières";
+CREATE SCHEMA "ville_Villeneuve-d'Ascq";

compose.yml

@@ -1,5 +1,3 @@
-version: "3.8"
-
 volumes:
   postgis_data:
 
@@ -32,7 +30,6 @@ services:
     environment:
       POSTGRES_HOST: postgis
       POSTGRES_DB: postgis
-      POSTGRES_SCHEMA: opendataindex
       POSTGRES_USER: postgis
       POSTGRES_PASSWORD: postgis
     volumes:
@@ -52,7 +49,7 @@ services:
       POSTGRES_HOST: postgis
       POSTGRES_PORT: 5432
       POSTGRES_DB: postgis
-      POSTGRES_SCHEMA: opendataindex
+      POSTGRES_SCHEMA: "ville_Villeneuve-d'Ascq"
       POSTGRES_USER: postgis
       POSTGRES_PASSWORD: postgis
       POSTGRES_POOL_MAXSIZE: 20

pom.xml

@@ -155,6 +155,20 @@
           <artifactId>maven-source-plugin</artifactId>
           <version>3.3.0</version>
         </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-failsafe-plugin</artifactId>
+          <version>3.2.1</version>
+          <executions>
+            <execution>
+              <goals>
+                <goal>integration-test</goal>
+                <goal>verify</goal>
+              </goals>
+            </execution>
+          </executions>
+          <inherited>true</inherited>
+        </plugin>
       </plugins>
     </pluginManagement>
     <plugins>

postgis.md

@@ -0,0 +1,59 @@
+## Configuration properties to define how to handle multiple PostgreSQL schemas to avoid Collection name clashes.
+
+When running with the `postgis` Spring profile, the default configuration is to serve data
+from all tables in all schemas, except for a few hard-coded schema names: `pg_toast`, `pg_catalog`, `information_schema`, `topology`, and `tiger`;
+and to prefix all feature type names with the schema name and a `:` delimiter.
+
+For example, a table `locations` in the `public` schema will be presented as `public:locations`.
+
+Newly created database schemas, or removed schemas, will be noticed after a period defined by the `postgis.schemas.refresh.interval` config property.
+
+If refresh is disabled (i.e. `postgis.schemas.refresh.enabled: false`), there will be no delay in noticing new or deleted PostgreSQL schemas,
+but the performance will suffer, as the list of schemas will need to be queried upon each API request.
+
+### Default configuration
+
+The default configuration (i.e. if nothing is explicitly configured) is equivalent to the following:
+
+```yaml
+postgis:
+  schemas:
+    delimiter: ":"
+    refresh:
+      enabled: true
+      interval: PT5S
+    include:
+    - schema: "*"
+      prefix-tables: true
+```
+
+### Schema aliasing
+
+Individual schema names can be aliased. For example, to change the prefix of the `ville_Villeneuve-d'Ascq` schema to `villeneuve`, use the following config:
+
+```yaml
+postgis:
+  schemas:
+    include:
+    - schema: "*"
+      prefix-tables: true
+    - schema: "ville_Villeneuve-d'Ascq"
+      alias: "villeneuve"
+```
+
+This will result in a table locations in that schema to be published as `villeneuve:locations` instead of `ville_Villeneuve-d'Ascq:locations`.
+
+A single schema can be configured to be un-prefixed, meaning its tables will be published as Collections without a schema prefix.
+
+For example, to remove the `public:` prefix for the `public` schema, use:
+
+```yaml
+postgis:
+  schemas:
+    include:
+    - schema: "public"
+      prefix-tables: false
+```
+
+If the `*` schema wildcard, or more than one schema is configured with `prefix-tables: false`, the application will fail to start.
+

src/services/ogc-features/pom.xml

@@ -107,6 +107,11 @@
               <artifactId>lombok</artifactId>
               <version>1.18.30</version>
             </path>
+            <path>
+              <groupId>org.springframework.boot</groupId>
+              <artifactId>spring-boot-configuration-processor</artifactId>
+              <version>3.3.1</version>
+            </path>
           </annotationProcessorPaths>
         </configuration>
       </plugin>
@@ -173,6 +178,10 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+      </plugin>
     </plugins>
   </build>
 

src/services/ogc-features/src/main/java/com/camptocamp/geotools/data/decorate/DecoratingDataStore.java

@@ -0,0 +1,18 @@
+package com.camptocamp.geotools.data.decorate;
+
+import org.geotools.api.data.DataStore;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Delegate;
+
+/**
+ * Abstract {@link DataStore} decorator forwarding all method calls to the
+ * decorated object
+ */
+@RequiredArgsConstructor
+public abstract class DecoratingDataStore implements DataStore {
+
+    @Delegate
+    protected final @NonNull DataStore delegate;
+}

src/services/ogc-features/src/main/java/com/camptocamp/geotools/data/decorate/DecoratingFeatureSource.java

@@ -0,0 +1,14 @@
+package com.camptocamp.geotools.data.decorate;
+
+import org.geotools.api.data.SimpleFeatureSource;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Delegate;
+
+@RequiredArgsConstructor
+public class DecoratingFeatureSource implements SimpleFeatureSource {
+
+    @Delegate
+    protected final @NonNull SimpleFeatureSource delegate;
+}

src/services/ogc-features/src/main/java/com/camptocamp/geotools/data/decorate/DecoratingReadOnlyDataStore.java

@@ -0,0 +1,66 @@
+package com.camptocamp.geotools.data.decorate;
+
+import static com.camptocamp.geotools.data.decorate.ReadOnlyDataStore.readOnlyException;
+
+import org.geotools.api.data.DataStore;
+import org.geotools.api.data.FeatureWriter;
+import org.geotools.api.data.Transaction;
+import org.geotools.api.feature.simple.SimpleFeature;
+import org.geotools.api.feature.simple.SimpleFeatureType;
+import org.geotools.api.feature.type.Name;
+import org.geotools.api.filter.Filter;
+
+import lombok.NonNull;
+
+/**
+ * A read-only {@link DecoratingDataStore}, where all mutating methods throw an
+ * {@link UnsupportedOperationException}
+ */
+public abstract class DecoratingReadOnlyDataStore extends DecoratingDataStore implements ReadOnlyDataStore {
+
+    protected DecoratingReadOnlyDataStore(@NonNull DataStore delegate) {
+        super(delegate);
+    }
+
+    @Override
+    public FeatureWriter<SimpleFeatureType, SimpleFeature> getFeatureWriter(String typeName, Filter filter,
+            Transaction transaction) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public FeatureWriter<SimpleFeatureType, SimpleFeature> getFeatureWriter(String typeName, Transaction transaction) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public FeatureWriter<SimpleFeatureType, SimpleFeature> getFeatureWriterAppend(String typeName,
+            Transaction transaction) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public void createSchema(SimpleFeatureType featureType) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public void updateSchema(Name typeName, SimpleFeatureType featureType) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public void removeSchema(Name typeName) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public void updateSchema(String typeName, SimpleFeatureType featureType) {
+        throw readOnlyException();
+    }
+
+    @Override
+    public void removeSchema(String typeName) {
+        throw readOnlyException();
+    }
+}

src/services/ogc-features/src/main/java/com/camptocamp/geotools/data/decorate/PrefixingDataStore.java

@@ -0,0 +1,38 @@
+package com.camptocamp.geotools.data.decorate;
+
+import static org.springframework.util.ObjectUtils.isEmpty;
+
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
+
+import org.geotools.api.data.DataStore;
+
+import lombok.NonNull;
+
+/**
+ * {@link RenamingDataStore} whose rename/undo functions apply or remove a given
+ * prefix to all FeatureType names
+ */
+public class PrefixingDataStore extends RenamingDataStore {
+
+    public PrefixingDataStore(@NonNull DataStore delegate, @NonNull Supplier</* Nullable */String> typeNamePrefix) {
+        super(delegate, rename(typeNamePrefix), undo(typeNamePrefix));
+    }
+
+    /**
+     * Function to apply the FeatureType name prefix as the renaming function of
+     * RenamingDataStore
+     */
+    private static @NonNull UnaryOperator<String> rename(Supplier<String> typeNamePrefix) {
+        return name -> isEmpty(typeNamePrefix.get()) ? name : "%s%s".formatted(typeNamePrefix.get(), name);
+    }
+
+    /**
+     * Function remote the FeatureType name prefix as the undo renaming function of
+     * RenamingDataStore
+     */
+    private static @NonNull UnaryOperator<String> undo(Supplier<String> typeNamePrefix) {
+        return prefixedName -> isEmpty(typeNamePrefix.get()) ? prefixedName
+                : prefixedName.substring(typeNamePrefix.get().length());
+    }
+}