Skip to content

Commit

Permalink
Run Azure blob store integration tests with Azure and Azurite
Browse files Browse the repository at this point in the history
Azurite is run as a testcontainer, and the github CI build runs all
integration tests for Java 11, 17, and 21.

In order to save on Azure resources, online integration tests are
run only if the Azurite based tests succeeded, using github secrets
to set up the account, key, and container; and for Java 11 only.
  • Loading branch information
groldan committed Aug 26, 2024
1 parent 0d2fc0b commit 31d0223
Show file tree
Hide file tree
Showing 23 changed files with 1,136 additions and 91 deletions.
80 changes: 80 additions & 0 deletions .github/workflows/azure-integration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
name: Azure BlobStore Integration

on:
push:
tags:
- "**"
pull_request:
paths:
- ".github/workflows/azure-integration.yml"
- "pom.xml"
- "geowebcache/pom.xml"
- "geowebcache/core/**"
- "geowebcache/azureblob/**"

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
azurite:
name: Azurite container
runs-on: ubuntu-latest
strategy:
matrix:
java-version: [ 11, 17, 21 ]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java-version }}
cache: 'maven'

- name: Tests against Azurite TestContainers
#-PexcludeOnlineTests includes Azurite container tests and excludes Azure online tests
run: |
mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
-DskipTests=true -DskipITs=false -B -ntp \
-PexcludeOnlineTests
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
azure:
name: Azure online
#if: github.repository == 'geowebcache/geowebcache'
runs-on: ubuntu-latest
needs: azurite
if: |
always() &&
!contains(needs.*.result, 'cancelled') &&
!contains(needs.*.result, 'failure')
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 11
cache: 'maven'

- name: Tests against Azure
env:
azure_account: ${{ secrets.AZURE_ACCOUNT }}
azure_account_key: ${{ secrets.AZURE_ACCOUNT_KEY }}
azure_container: ${{ secrets.AZURE_CONTAINER }}
if: ${{ env.azure_account != null }} && ${{ env.azure_account_key != null }}
run: | #-PexcludeDockerTests includes Azure online tests and excludes Azurite container tests
echo "accountName=$azure_account" > $HOME/.gwc_azure_tests.properties
echo "accountKey=$azure_account_key" >> $HOME/.gwc_azure_tests.properties
echo "container=$azure_container" >> $HOME/.gwc_azure_tests.properties
echo 'maxConnections=8' >> $HOME/.gwc_azure_tests.properties
echo 'useHTTPS=true' >> $HOME/.gwc_azure_tests.properties
mvn verify -f geowebcache/pom.xml -pl :gwc-azure-blob -am \
-DskipTests=true -DskipITs=false -B -ntp \
-PexcludeDockerTests
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
9 changes: 6 additions & 3 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Build with Maven
run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml
# -DskipITs skips failsafe integration tests but runs unit tests with surefire. Integration tests have their own jobs
run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml -DskipITs
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
Expand All @@ -49,7 +50,8 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Build with Maven
run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml
# -DskipITs skips failsafe integration tests but runs unit tests with surefire. Integration tests have their own jobs
run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml -DskipITs
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
Expand All @@ -71,7 +73,8 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Build with Maven
run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml
# -DskipITs skips failsafe integration tests but runs unit tests with surefire. Integration tests have their own jobs
run: mvn -B clean install -Dspotless.apply.skip=true -Dall -T2 --file geowebcache/pom.xml -DskipITs
- name: Remove SNAPSHOT jars from repository
run: |
find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {}
Expand Down
65 changes: 65 additions & 0 deletions geowebcache/azureblob/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,70 @@
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<!-- skips the integration tests against a real Azure account, used to split up the CI builds-->
<id>excludeOnlineTests</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<excludes>
<exclude>org.geowebcache.azure.tests.online.*IT</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<!-- skips the integration tests against the Azurite test container, used to split up the CI builds-->
<id>excludeDockerTests</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<excludes>
<exclude>org.geowebcache.azure.tests.container.*IT</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
* Azure Blobstore type-resolved data from a {@link AzureBlobStoreInfo} using enviroment variables
* if enabled.
*/
class AzureBlobStoreData {
public class AzureBlobStoreData {

private String container;
private String prefix;
Expand All @@ -36,7 +36,7 @@ class AzureBlobStoreData {
private String proxyPassword;
private String serviceURL;

AzureBlobStoreData() {}
public AzureBlobStoreData() {}

public AzureBlobStoreData(
final AzureBlobStoreInfo storeInfo, final GeoWebCacheEnvironment environment) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
import org.geowebcache.util.URLs;
import org.springframework.http.HttpStatus;

class AzureClient implements Closeable {
public class AzureClient implements Closeable {

private final NettyClient.Factory factory;
private AzureBlobStoreData configuration;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,25 @@
import java.util.stream.Stream;
import org.easymock.EasyMock;
import org.geowebcache.GeoWebCacheException;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreConformanceIT;
import org.geowebcache.layer.TileLayer;
import org.geowebcache.layer.TileLayerDispatcher;
import org.geowebcache.locks.LockProvider;
import org.geowebcache.locks.NoOpLockProvider;
import org.geowebcache.storage.AbstractBlobStoreTest;
import org.junit.Assume;
import org.junit.Rule;

public class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {
public PropertiesLoader testConfigLoader = new PropertiesLoader();
/**
* @see OnlineAzureBlobStoreConformanceIT
* @see AzuriteAzureBlobStoreConformanceIT
*/
public abstract class AzureBlobStoreConformanceTest extends AbstractBlobStoreTest<AzureBlobStore> {

@Rule
public TemporaryAzureFolder tempFolder =
new TemporaryAzureFolder(testConfigLoader.getProperties());
protected abstract AzureBlobStoreData getConfiguration();

@Override
public void createTestUnit() throws Exception {
Assume.assumeTrue(tempFolder.isConfigured());
AzureBlobStoreData config = tempFolder.getConfig();
AzureBlobStoreData config = getConfiguration();

TileLayerDispatcher layers = createMock(TileLayerDispatcher.class);
LockProvider lockProvider = new NoOpLockProvider();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
import java.util.Map;
import java.util.logging.Logger;
import org.geotools.util.logging.Logging;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreIntegrationIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreIntegrationIT;
import org.geowebcache.config.DefaultGridsets;
import org.geowebcache.grid.GridSet;
import org.geowebcache.grid.GridSetBroker;
Expand Down Expand Up @@ -69,19 +71,20 @@
* Integration tests for {@link AzureBlobStore}.
*
* <p>This is an abstract class for both online and offline integration tests.
*
* @see OnlineAzureBlobStoreIntegrationIT
* @see AzuriteAzureBlobStoreIntegrationIT
*/
public abstract class AbstractAzureBlobStoreIntegrationTest {
public abstract class AzureBlobStoreIntegrationTest {

private static Logger log = Logging.getLogger(PropertiesLoader.class.getName());
private static Logger log = Logging.getLogger(AzureBlobStoreIntegrationTest.class.getName());

private static final String DEFAULT_FORMAT = "png";

private static final String DEFAULT_GRIDSET = "EPSG:4326";

private static final String DEFAULT_LAYER = "topp:world";

public PropertiesLoader testConfigLoader = new PropertiesLoader();

private AzureBlobStore blobStore;

protected abstract AzureBlobStoreData getConfiguration();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItemInArray;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeFalse;

import io.reactivex.Flowable;
import java.nio.ByteBuffer;
import org.easymock.EasyMock;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT;
import org.geowebcache.layer.TileLayerDispatcher;
import org.geowebcache.locks.LockProvider;
import org.geowebcache.locks.NoOpLockProvider;
Expand All @@ -30,23 +31,14 @@
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.experimental.theories.DataPoints;
import org.junit.experimental.theories.Theories;
import org.junit.runner.RunWith;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;
import org.springframework.http.HttpStatus;

@RunWith(AzureBlobStoreSuitabilityTest.MyTheories.class)
public class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {

public PropertiesLoader testConfigLoader = new PropertiesLoader();

@Rule
public TemporaryAzureFolder tempFolder =
new TemporaryAzureFolder(testConfigLoader.getProperties());
/**
* @see OnlineAzureBlobStoreSuitabilityIT
* @see AzuriteAzureBlobStoreSuitabilityIT
*/
public abstract class AzureBlobStoreSuitabilityTest extends BlobStoreSuitabilityTest {

@DataPoints
public static String[][] persistenceLocations = {
Expand All @@ -67,6 +59,10 @@ public void setup() throws Exception {
EasyMock.replay(tld);
}

protected abstract AzureBlobStoreData getConfiguration();

protected abstract AzureClient getClient();

@SuppressWarnings("unchecked")
@Override
protected Matcher<Object> existing() {
Expand All @@ -81,13 +77,12 @@ protected Matcher<Object> empty() {

@Override
public BlobStore create(Object dir) throws Exception {
AzureBlobStoreData info = tempFolder.getConfig();
AzureBlobStoreData info = getConfiguration();
for (String path : (String[]) dir) {
String fullPath = info.getPrefix() + "/" + path;
ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes());
int statusCode =
tempFolder
.getClient()
getClient()
.getBlockBlobURL(fullPath)
.upload(Flowable.just(byteBuffer), byteBuffer.limit())
.blockingGet()
Expand All @@ -96,27 +91,4 @@ public BlobStore create(Object dir) throws Exception {
}
return new AzureBlobStore(info, tld, locks);
}

// Sorry, this bit of evil makes the Theories runner gracefully ignore the
// tests if Azure is unavailable. There's probably a better way to do this.
public static class MyTheories extends Theories {

public MyTheories(Class<?> klass) throws InitializationError {
super(klass);
}

@Override
public Statement methodBlock(FrameworkMethod method) {
if (new PropertiesLoader().getProperties().containsKey("container")) {
return super.methodBlock(method);
} else {
return new Statement() {
@Override
public void evaluate() {
assumeFalse("Azure unavailable", true);
}
};
}
}
}
}
Loading

0 comments on commit 31d0223

Please sign in to comment.