Skip to content

Commit

Permalink
Merge pull request #13 from camptocamp/bug/recover_from_schema_changes
Browse files Browse the repository at this point in the history 
Ensure the PostgGIS backend supports external schema changes
  • Loading branch information
@groldan
groldan authored Oct 31, 2023
2 parents e3aaf97 + a077a37 commit 9608f7c
Show file tree
Hide file tree
Showing 9 changed files with 476 additions and 53 deletions.
47 changes: 40 additions & 7 deletions ...ptocamp/opendata/ogc/features/autoconfigure/geotools/PostgisBackendAutoConfiguration.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package com.camptocamp.opendata.ogc.features.autoconfigure.geotools;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;

import javax.sql.DataSource;

import org.geotools.api.data.DataStore;
import org.geotools.data.postgis.PostgisNGDataStoreFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.geotools.jdbc.JDBCDataStore;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.context.annotation.Bean;
Expand All @@ -18,22 +20,26 @@

import com.camptocamp.opendata.ogc.features.repository.CollectionRepository;
import com.camptocamp.opendata.ogc.features.repository.DataStoreCollectionRepository;
import com.camptocamp.opendata.ogc.features.repository.DataStoreProvider;
import com.camptocamp.opendata.ogc.features.repository.DefaultDataStoreProvider;
import com.camptocamp.opendata.producer.geotools.FeatureToRecord;

import lombok.NonNull;

@AutoConfiguration
@Profile("postgis")
public class PostgisBackendAutoConfiguration implements WebMvcConfigurer {

@Bean
CollectionRepository postgisDataStoreCollectionRepository(@Qualifier("indexDataStore") DataStore indexStore) {
return new DataStoreCollectionRepository(indexStore, new FeatureToRecord());
CollectionRepository postgisDataStoreCollectionRepository(DataStoreProvider dsProvider) {
return new DataStoreCollectionRepository(dsProvider, new FeatureToRecord());
}

@Bean(name = "indexDataStore")
@DependsOn("databaseStartupValidator")
DataStore postgisDataStore(DataSource dataSource, @Value("${pg.schema:opendataindex}") String schema)
DataStoreProvider postgisDataStore(DataSource dataSource, @Value("${pg.schema:opendataindex}") String schema)
throws IOException {
Map<String, ?> params = Map.of(//
Map<String, Object> params = Map.of(//
PostgisNGDataStoreFactory.DBTYPE.key, "postgis", //
PostgisNGDataStoreFactory.DATASOURCE.key, dataSource, //
PostgisNGDataStoreFactory.SCHEMA.key, schema, //
Expand All @@ -43,8 +49,7 @@ DataStore postgisDataStore(DataSource dataSource, @Value("${pg.schema:opendatain
PostgisNGDataStoreFactory.LOOSEBBOX.key, true//
);

PostgisNGDataStoreFactory fac = new PostgisNGDataStoreFactory();
return fac.createDataStore(params);
return new PostgisDataStoreProvider(params);
}

@Bean
Expand All @@ -53,4 +58,32 @@ DatabaseStartupValidator databaseStartupValidator(DataSource dataSource) {
dsv.setDataSource(dataSource);
return dsv;
}

public static class PostgisDataStoreProvider extends DefaultDataStoreProvider {

public PostgisDataStoreProvider(@NonNull Map<String, Object> connectionParams) {
super(new HashMap<>(connectionParams));
}

public DataSource getDataSource() {
return (DataSource) super.connectionParams.get(PostgisNGDataStoreFactory.DATASOURCE.key);
}

public void setDataSource(DataSource ds) {
super.connectionParams.put(PostgisNGDataStoreFactory.DATASOURCE.key, ds);
if (null != super.store) {
((JDBCDataStore) super.store).setDataSource(ds);
}
}

@Override
protected @NonNull DataStore create() {
PostgisNGDataStoreFactory fac = new PostgisNGDataStoreFactory();
try {
return fac.createDataStore(connectionParams);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
}
72 changes: 49 additions & 23 deletions ...camp/opendata/ogc/features/autoconfigure/geotools/SampleDataBackendAutoConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
Expand All @@ -19,7 +20,6 @@
import org.geotools.geopkg.GeoPkgDataStoreFactory;
import org.geotools.jdbc.JDBCDataStore;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Profile;
Expand All @@ -28,10 +28,10 @@

import com.camptocamp.opendata.ogc.features.repository.CollectionRepository;
import com.camptocamp.opendata.ogc.features.repository.DataStoreCollectionRepository;
import com.camptocamp.opendata.ogc.features.repository.DataStoreProvider;
import com.camptocamp.opendata.producer.geotools.FeatureToRecord;
import com.google.common.io.ByteStreams;

import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

/**
Expand All @@ -47,19 +47,9 @@
@Slf4j
public class SampleDataBackendAutoConfiguration {

// @Bean
// IndexedReader sampleDataIndexedReader(@NonNull GeoToolsDataReader gtReader) throws IOException {
// return new SampleDataReader(gtReader);
// }

@Bean
CollectionRepository sampleDataDataStoreCollectionRepository(@Qualifier("indexDataStore") DataStore indexStore) {
return new DataStoreCollectionRepository(indexStore, new FeatureToRecord());
}

@Bean
DataStore indexDataStore(SampleData sampleData) throws IOException {
return sampleData.getDataStore();
CollectionRepository sampleDataDataStoreCollectionRepository(SampleData dsProvider) {
return new DataStoreCollectionRepository(dsProvider, new FeatureToRecord());
}

@Bean
Expand All @@ -72,19 +62,47 @@ SampleData sampleData() throws IOException {
* shutdown, since the geotools CSV datastore does not support URL resources,
* only Files
*/
private static class SampleData implements DisposableBean {
private static class SampleData implements DataStoreProvider, DisposableBean {

private Path tempDirectory;

private DataStore dataStore;

@Override
public DataStore get() {
if (null == dataStore) {
synchronized (this) {
if (null == dataStore) {
try {
create();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
}
return dataStore;
}

private Path tempDirectory = Files.createTempDirectory("ogc-features-sample-data");
private final @Getter DataStore dataStore;
@Override
public void reInit() {
try {
dispose();
create();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

SampleData() throws IOException {
private void create() throws IOException {
tempDirectory = Files.createTempDirectory("ogc-features-sample-data");
dataStore = new MemoryDataStore();
log.info("Extracting sample data to {}", tempDirectory);

File sd = copyToTempDir("sample-datasets.gpkg");

final GeoPkgDataStoreFactory factory = new GeoPkgDataStoreFactory();
final Map<String, ?> params = Map.of(//
Map<String, Object> params = Map.of(//
GeoPkgDataStoreFactory.DBTYPE.key, "geopkg", //
GeoPkgDataStoreFactory.DATABASE.key, sd, //
// Whether to return only tables listed as features in gpkg_contents, or give
Expand Down Expand Up @@ -113,10 +131,18 @@ private static class SampleData implements DisposableBean {

@Override
public void destroy() throws Exception {
dataStore.dispose();
if (tempDirectory != null && Files.isDirectory(tempDirectory)) {
log.info("Deleting sample data directory {}", tempDirectory);
FileSystemUtils.deleteRecursively(tempDirectory);
dispose();
}

private void dispose() throws IOException {
if (null != dataStore) {
dataStore.dispose();
if (tempDirectory != null && Files.isDirectory(tempDirectory)) {
log.info("Deleting sample data directory {}", tempDirectory);
FileSystemUtils.deleteRecursively(tempDirectory);
}
dataStore = null;
tempDirectory = null;
}
}

Expand Down
1 change: 1 addition & 0 deletions ...s/src/main/java/com/camptocamp/opendata/ogc/features/model/GeoToolsFeatureCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.geotools.data.DataUtilities;
import org.geotools.data.simple.SimpleFeatureCollection;
import org.geotools.data.simple.SimpleFeatureIterator;
import org.geotools.jdbc.JDBCDataStore;

import com.camptocamp.opendata.model.GeodataRecord;
import com.camptocamp.opendata.producer.geotools.FeatureToRecord;
Expand Down
90 changes: 67 additions & 23 deletions ...n/java/com/camptocamp/opendata/ogc/features/repository/DataStoreCollectionRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.function.Function;

import org.geotools.api.data.DataStore;
Expand All @@ -31,20 +32,25 @@

import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

@RequiredArgsConstructor
@Slf4j
public class DataStoreCollectionRepository implements CollectionRepository {

// private final @NonNull Producers producers;
private final @NonNull DataStore dataStore;
private final @NonNull DataStoreProvider dataStoreProvider;
private final @NonNull Function<SimpleFeature, GeodataRecord> featureMapper;

private final FilterFactory ff = CommonFactoryFinder.getFilterFactory();

private DataStore dataStore() {
return dataStoreProvider.get();
}

@Override
public List<Collection> getCollections() {
try {
String[] typeNames = dataStore.getTypeNames();
String[] typeNames = dataStore().getTypeNames();
return Arrays.stream(typeNames).map(this::loadCollection).toList();
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand All @@ -62,34 +68,72 @@ public Optional<Collection> findCollection(String collectionId) {

@Override
public Optional<GeodataRecord> getRecord(String collectionId, String featureId) {
SimpleFeature feature;

Query query = new Query(collectionId, ff.id(ff.featureId(featureId)));
SimpleFeatureCollection features = query(query);
try (SimpleFeatureIterator it = features.features()) {
feature = it.hasNext() ? it.next() : null;
}
final Query query = new Query(collectionId, ff.id(ff.featureId(featureId)));

return runWithRetry("getRecord(%s:%s)".formatted(collectionId, featureId), () -> {
SimpleFeatureCollection features = query(query);
SimpleFeature feature;
try (SimpleFeatureIterator it = features.features()) {
feature = it.hasNext() ? it.next() : null;
}

return Optional.ofNullable(feature).map(featureMapper);
return Optional.ofNullable(feature).map(featureMapper);
});
}

private <T> T runWithRetry(String description, Callable<T> command) {
try {
return command.call();
} catch (Exception e) {
log.info("Retrying command %s".formatted(description));
dataStoreProvider.reInit();
try {
return command.call();
} catch (Exception e2) {
log.info("Retry command failed. Giving up for %s".formatted(description));
if (e2 instanceof RuntimeException) {
throw (RuntimeException) e2;
}
throw new RuntimeException(e);
}
}
}

@Override
public FeatureCollection query(@NonNull DataQuery query) {
Collection collection = findCollection(query.getLayerName()).orElseThrow();
Query gtQuery = toQuery(query);
SimpleFeatureCollection features = query(gtQuery);

long matched = count(toQuery(query.withLimit(null)));
long returned = count(gtQuery);
GeoToolsFeatureCollection ret = new GeoToolsFeatureCollection(collection, features);
ret.setNumberMatched(matched);
ret.setNumberReturned(returned);
return ret;
final Collection collection = findCollection(query.getLayerName()).orElseThrow();
final Query gtQuery = toQuery(query);
return runWithRetry("query(%s)".formatted(query.getLayerName()), () -> {
ensureSchemaIsInSync(gtQuery);
SimpleFeatureCollection fc = query(gtQuery);
long matched = count(toQuery(query.withLimit(null)));
long returned = count(gtQuery);
GeoToolsFeatureCollection ret = new GeoToolsFeatureCollection(collection, fc);
ret.setNumberMatched(matched);
ret.setNumberReturned(returned);
return ret;
});
}

/**
* Workaround to make sure the datastore cached featuretype is in sync with the
* one in the database in case it has changed under the hood
*/
private void ensureSchemaIsInSync(Query gtQuery) {
Query noopQuery = new Query(gtQuery);
noopQuery.setMaxFeatures(0);
SimpleFeatureCollection fc = query(noopQuery);
try (SimpleFeatureIterator it = fc.features()) {

} catch (RuntimeException e) {
throw e;
}
}

private int count(Query query) {
try {
return dataStore.getFeatureSource(query.getTypeName()).getCount(query);
return dataStore().getFeatureSource(query.getTypeName()).getCount(query);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Expand All @@ -114,7 +158,7 @@ private Query toQuery(@NonNull DataQuery query) {
private SimpleFeatureCollection query(Query query) {
try {
String typeName = query.getTypeName();
SimpleFeatureSource featureSource = dataStore.getFeatureSource(typeName);
SimpleFeatureSource featureSource = dataStore().getFeatureSource(typeName);
return featureSource.getFeatures(query);
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand All @@ -126,7 +170,7 @@ private Collection loadCollection(String typeName) {
c.setTitle(typeName);
SimpleFeatureType schema;
try {
schema = dataStore.getSchema(typeName);
schema = dataStore().getSchema(typeName);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Expand Down
10 changes: 10 additions & 0 deletions ...ures/src/main/java/com/camptocamp/opendata/ogc/features/repository/DataStoreProvider.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.camptocamp.opendata.ogc.features.repository;

import org.geotools.api.data.DataStore;

public interface DataStoreProvider {

DataStore get();

public void reInit();
}
Loading

0 comments on commit 9608f7c

Please sign in to comment.