Skip to content

Commit

Permalink
Merge pull request #1316 from GeoWebCache/azureblob_upgrade_12.x
Browse files Browse the repository at this point in the history
Upgrade Azure Blobstore from legacy azure-sdk (11.0) to latest (12.27.1)
petersmythe authored Sep 18, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents c3ffd5e + ad540d8 commit ea0fd79
Showing 15 changed files with 474 additions and 905 deletions.
25 changes: 22 additions & 3 deletions geowebcache/azureblob/pom.xml
Original file line number Diff line number Diff line change
@@ -11,6 +11,26 @@
<groupId>org.geowebcache</groupId>
<artifactId>gwc-azure-blob</artifactId>

<properties>
<!-- Same sdk version as imageio-ext's cog-ragengereader-azure for GeoSever compatibility -->
<azure.version>12.27.1</azure.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<!--
Use the same netty version as imageio-ext's cog-ragengereader-s3 and cog-rangereader-azure
See https://github.com/geosolutions-it/imageio-ext/pull/312
-->
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.113.Final</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.geowebcache</groupId>
@@ -19,10 +39,9 @@
</dependency>

<dependency>
<groupId>com.microsoft.azure</groupId>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<!-- at the time of writing 11.0.1 was available but pom on repoes was corrupted -->
<version>11.0.0</version>
<version>${azure.version}</version>
</dependency>

<dependency>

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ public class AzureBlobStoreData {
private String accountName;
private String accountKey;
private Integer maxConnections;
private Boolean useHTTPS;
private boolean useHTTPS;
private String proxyHost;
private Integer proxyPort;
private String proxyUsername;
@@ -113,11 +113,11 @@ public void setMaxConnections(Integer maxConnections) {
this.maxConnections = maxConnections;
}

public Boolean isUseHTTPS() {
public boolean isUseHTTPS() {
return useHTTPS;
}

public void setUseHTTPS(Boolean useHTTPS) {
public void setUseHTTPS(boolean useHTTPS) {
this.useHTTPS = useHTTPS;
}

@@ -137,6 +137,7 @@ public void setProxyPort(Integer proxyPort) {
this.proxyPort = proxyPort;
}

/** unused */
public String getProxyUsername() {
return proxyUsername;
}
@@ -145,6 +146,7 @@ public void setProxyUsername(String proxyUsername) {
this.proxyUsername = proxyUsername;
}

/** unused */
public String getProxyPassword() {
return proxyPassword;
}
Original file line number Diff line number Diff line change
@@ -52,7 +52,7 @@ public class AzureBlobStoreInfo extends BlobStoreInfo {

private String maxConnections;

private Boolean useHTTPS = true;
private boolean useHTTPS = true;

private String proxyHost;

@@ -139,7 +139,7 @@ public Boolean isUseHTTPS() {
}

/** @param useHTTPS whether to use HTTPS (true) or HTTP (false) when talking to Azure */
public void setUseHTTPS(Boolean useHTTPS) {
public void setUseHTTPS(boolean useHTTPS) {
this.useHTTPS = useHTTPS;
}

@@ -260,7 +260,7 @@ public int hashCode() {
result = prime * result + ((proxyPort == null) ? 0 : proxyPort.hashCode());
result = prime * result + ((proxyUsername == null) ? 0 : proxyUsername.hashCode());
result = prime * result + ((serviceURL == null) ? 0 : serviceURL.hashCode());
result = prime * result + ((useHTTPS == null) ? 0 : useHTTPS.hashCode());
result = prime * result + (useHTTPS ? 1 : 0);
return result;
}

@@ -300,9 +300,7 @@ public boolean equals(Object obj) {
if (serviceURL == null) {
if (other.serviceURL != null) return false;
} else if (!serviceURL.equals(other.serviceURL)) return false;
if (useHTTPS == null) {
if (other.useHTTPS != null) return false;
} else if (!useHTTPS.equals(other.useHTTPS)) return false;
if (useHTTPS != other.useHTTPS) return false;
return true;
}

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -96,7 +96,7 @@ public void before() throws Exception {
TileLayerDispatcher layers = mock(TileLayerDispatcher.class);
LockProvider lockProvider = new NoOpLockProvider();
TileLayer layer = mock(TileLayer.class);
when(layers.getTileLayer(eq(DEFAULT_LAYER))).thenReturn(layer);
when(layers.getTileLayer(DEFAULT_LAYER)).thenReturn(layer);
when(layer.getName()).thenReturn(DEFAULT_LAYER);
when(layer.getId()).thenReturn(DEFAULT_LAYER);
blobStore = new AzureBlobStore(config, layers, lockProvider);
Original file line number Diff line number Diff line change
@@ -18,8 +18,11 @@
import static org.hamcrest.Matchers.hasItemInArray;
import static org.junit.Assert.assertTrue;

import io.reactivex.Flowable;
import java.nio.ByteBuffer;
import com.azure.core.http.rest.Response;
import com.azure.core.util.BinaryData;
import com.azure.core.util.Context;
import com.azure.storage.blob.models.BlockBlobItem;
import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions;
import org.easymock.EasyMock;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreSuitabilityIT;
import org.geowebcache.azure.tests.online.OnlineAzureBlobStoreSuitabilityIT;
@@ -63,13 +66,13 @@ public void setup() throws Exception {

protected abstract AzureClient getClient();

@SuppressWarnings("unchecked")
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
protected Matcher<Object> existing() {
return (Matcher) hasItemInArray(equalTo("metadata.properties"));
}

@SuppressWarnings("unchecked")
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
protected Matcher<Object> empty() {
return (Matcher) Matchers.emptyArray();
@@ -80,14 +83,13 @@ public BlobStore create(Object dir) throws Exception {
AzureBlobStoreData info = getConfiguration();
for (String path : (String[]) dir) {
String fullPath = info.getPrefix() + "/" + path;
ByteBuffer byteBuffer = ByteBuffer.wrap("testAbc".getBytes());
int statusCode =
BinaryData data = BinaryData.fromString("testAbc");
Response<BlockBlobItem> response =
getClient()
.getBlockBlobURL(fullPath)
.upload(Flowable.just(byteBuffer), byteBuffer.limit())
.blockingGet()
.statusCode();
assertTrue(HttpStatus.valueOf(statusCode).is2xxSuccessful());
.getBlockBlobClient(fullPath)
.uploadWithResponse(
new BlockBlobSimpleUploadOptions(data), null, Context.NONE);
assertTrue(HttpStatus.valueOf(response.getStatusCode()).is2xxSuccessful());
}
return new AzureBlobStore(info, tld, locks);
}
Original file line number Diff line number Diff line change
@@ -30,14 +30,8 @@
*/
public class AzuriteAzureBlobStoreConformanceIT extends AzureBlobStoreConformanceTest {

/**
* Use "legacy" container to work with {@literal
* com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as
* AzuriteContainer.legacy().debugLegacy() to print out request/response information for
* debugging purposes
*/
@ClassRule
public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker();
public static AzuriteContainer azurite = AzuriteContainer.latest().disabledWithoutDocker();

/** Used to get a per-test case Azure container */
@Rule public TestName testName = new TestName();
Original file line number Diff line number Diff line change
@@ -30,14 +30,8 @@
*/
public class AzuriteAzureBlobStoreIntegrationIT extends AzureBlobStoreIntegrationTest {

/**
* Use "legacy" container to work with {@literal
* com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as
* AzuriteContainer.legacy().debugLegacy() to print out request/response information for
* debugging purposes
*/
@ClassRule
public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker();
public static AzuriteContainer azurite = AzuriteContainer.latest().disabledWithoutDocker();

/** Used to get a per-test case Azure container */
@Rule public TestName testName = new TestName();
Original file line number Diff line number Diff line change
@@ -34,14 +34,8 @@
*/
public class AzuriteAzureBlobStoreSuitabilityIT extends AzureBlobStoreSuitabilityTest {

/**
* Use "legacy" container to work with {@literal
* com.microsoft.azure:azure-storage-blob:jar:11.0.0}. Instantiate it as
* AzuriteContainer.legacy().debugLegacy() to print out request/response information for
* debugging purposes
*/
@ClassRule
public static AzuriteContainer azurite = AzuriteContainer.legacy().disabledWithoutDocker();
public static AzuriteContainer azurite = AzuriteContainer.latest().disabledWithoutDocker();

/** Used to get a per-test case Azure container */
@Rule public TestName testName = new TestName();
Original file line number Diff line number Diff line change
@@ -21,7 +21,6 @@
import org.junit.Assume;
import org.junit.Rule;
import org.junit.Test;
import org.springframework.http.HttpStatus;

public class OnlineAzureBlobStoreIntegrationIT extends AzureBlobStoreIntegrationTest {

@@ -41,14 +40,8 @@ protected AzureBlobStoreData getConfiguration() {
public void testCreatesStoreMetadataOnStart() {
String prefix = tempFolder.getConfig().getPrefix();
// if the file does not exist a StorageException will be thrown
int status =
tempFolder
.getClient()
.getBlockBlobURL(prefix + "/metadata.properties")
.getProperties()
.blockingGet()
.statusCode();
assertTrue(
"Expected success but got " + status, HttpStatus.valueOf(status).is2xxSuccessful());
String key = prefix + "/metadata.properties";
boolean exists = tempFolder.getClient().getBlockBlobClient(key).exists();
assertTrue("blob " + key + " does not exist", exists);
}
}
Original file line number Diff line number Diff line change
@@ -17,15 +17,14 @@
import static com.google.common.base.Preconditions.checkState;
import static org.junit.Assert.assertTrue;

import com.microsoft.azure.storage.blob.BlockBlobURL;
import com.microsoft.azure.storage.blob.models.BlobItem;
import java.util.List;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.specialized.BlockBlobClient;
import java.util.Properties;
import java.util.UUID;
import java.util.stream.Stream;
import org.geowebcache.azure.AzureBlobStoreData;
import org.geowebcache.azure.AzureClient;
import org.junit.rules.ExternalResource;
import org.springframework.http.HttpStatus;

/**
* The TemporaryAzureFolder provides a path prefix for Azure storage and deletes all resources under
@@ -71,7 +70,6 @@ protected void after() {
delete();
} finally {
temporaryPrefix = null;
client.close();
}
}

@@ -117,13 +115,14 @@ public void delete() {
return;
}

List<BlobItem> blobs = client.listBlobs(temporaryPrefix, Integer.MAX_VALUE);
for (BlobItem blob : blobs) {
BlockBlobURL blockBlobURL = client.getBlockBlobURL(blob.name());
int status = blockBlobURL.delete().blockingGet().statusCode();
assertTrue(
"Expected success but got " + status + " while deleting " + blob.name(),
HttpStatus.valueOf(status).is2xxSuccessful());
try (Stream<BlobItem> blobs = client.listBlobs(temporaryPrefix)) {
blobs.forEach(
blob -> {
BlockBlobClient blockBlobURL = client.getBlockBlobClient(blob.getName());
assertTrue(
"Expected success while deleting " + blob.getName(),
blockBlobURL.deleteIfExists());
});
}
}

Original file line number Diff line number Diff line change
@@ -18,9 +18,7 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;

import java.io.IOException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.UncheckedException;
import org.geowebcache.azure.AzureBlobStoreData;
import org.geowebcache.azure.tests.container.AzuriteAzureBlobStoreConformanceIT;
import org.junit.Assume;
@@ -33,14 +31,15 @@

/**
* <a href="https://java.testcontainers.org/">Testcontainers</a> container for AWS <a href=
* "https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite">Azurite</a>
* blobstore test environment.
* "https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite">Azurite</a> Azure
* Storage Emulator.
*
* <p>Runs the <a href=
* "https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=docker-hub%2Cblob-storage">Azurite
* Docker image</a> for local Azure Storage development with testcontainers.
*
* <p>Azurite accepts the same well-known account and key used by the legacy Azure Storage Emulator:
* <p>Azurite sets up the following well-known account and key for the Azure Storage Emulator,
* available through {@link #getAccountName()} and {@link #getAccountKey()}:
*
* <ul>
* <li>Account name: {@code devstoreaccount1}
@@ -52,15 +51,7 @@
*
* <pre>
* <code>
* @Rule public AzuriteContainer azurite = AzuriteContainer.legacy();
* </code>
* </pre>
*
* works with the old {@code com.microsoft.azure:azure-storage-blob:jar:11.0.0} as a dependency.
*
* <pre>
* <code>
* @Rule public AzuriteContainer azurite = AzuriteContainer.legacy();
* @Rule public AzuriteContainer azurite = AzuriteContainer.latest();
* </code>
* </pre>
*
@@ -70,7 +61,7 @@
*
* <pre>
* <code>
* @ClassRule public static AzuriteContainer azurite = AzuriteContainer.legacy();
* @ClassRule public static AzuriteContainer azurite = AzuriteContainer.latest();
*
* @Test
* public void azureBlobStoreSmokeTest(){
@@ -87,78 +78,24 @@ public class AzuriteContainer extends GenericContainer<AzuriteContainer> {
private static final DockerImageName LATEST_IMAGE =
DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite:latest");

private static final DockerImageName LEGACY_IMAGE =
DockerImageName.parse("arafato/azurite:2.6.5");

private final String accountName = "devstoreaccount1";
private final String accountKey =
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";

private final int blobsPort = 10_000;

private AzuriteContainerLegacyProxy proxy;

private final boolean doProxy;

/** Whether to print request/response debug information when in {@link #legacy} mode */
private boolean debugRequests;

/** flag for {@link #disabledWithoutDocker()} */
private boolean disabledWithoutDocker;

private AzuriteContainer(DockerImageName imageName, boolean doProxy) {
private AzuriteContainer(DockerImageName imageName) {
super(imageName);
this.doProxy = doProxy;
super.setWaitStrategy(Wait.forListeningPort());
super.addExposedPort(blobsPort);
}

/**
* @return a container running {@code arafato/azurite:2.6.5} and {@link #getBlobsPort() proxied}
* to fix protocol discrepancies so it works correctly with older {@code
* com.microsoft.azure:azure-storage-blob} dependencies
*/
public static AzuriteContainer legacy() {
return new AzuriteContainer(LEGACY_IMAGE, true);
}

/** @return a container running {@code mcr.microsoft.com/azure-storage/azurite:latest} */
public static AzuriteContainer latest() {
return new AzuriteContainer(LATEST_IMAGE, false);
}

/**
* Enables request/response debugging when in legacy mode
*
* <p>Sample output:
*
* <pre>
* <code>
* routing GET http://localhost:44445/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png to GET http://localhost:33319/devstoreaccount1/testputgetblobisnotbytearrayresource/topp%3Aworld%2FEPSG%3A4326%2Fpng%2Fdefault%2F12%2F20%2F30.png
* applied request header Authorization: SharedKey devstoreaccount1:6UeSk1Qf8XRibLI1sE3tasmDxOtVxGUSMDQqRUDIW9Y=
* applied request header x-ms-version: 2018-11-09
* applied request header x-ms-date: Fri, 09 Aug 2024 17:08:38 GMT
* applied request header host: localhost
* applied request header x-ms-client-request-id: 526b726a-13af-49a3-b277-fdf645d77903
* applied request header User-Agent: Azure-Storage/11.0.0 (JavaJRE 11.0.23; Linux 6.8.0-39-generic)
* response: 200 OK
* applied response header X-Powered-By: Express
* applied response header ETag: "jzUOHaHcch36ue3TFspQaLiWSvo"
* applied response header Last-Modified: Fri, 09 Aug 2024 17:08:38 GMT
* applied response header x-ms-version: 2016-05-31
* applied response header date: Fri, 09 Aug 2024 17:08:38 GMT
* applied response header x-ms-request-id: 05130dd1-5672-11ef-a96b-c7f08f042b95
* applied response header accept-ranges: bytes
* applied response header x-ms-blob-type: BlockBlob
* applied response header x-ms-request-server-encrypted: false
* applied response header Content-Type: image/png
* Content-Type: image/png
* </code>
* </pre>
*/
public AzuriteContainer debugLegacy() {
this.debugRequests = true;
return this;
return new AzuriteContainer(LATEST_IMAGE);
}

/**
@@ -189,32 +126,6 @@ public Statement apply(Statement base, Description description) {
return super.apply(base, description);
}

@Override
public void start() {
super.start();
if (doProxy && proxy == null) {
int targetPort = getRealBlobsPort();
proxy = new AzuriteContainerLegacyProxy(targetPort).debugRequests(debugRequests);
try {
proxy.start();
} catch (IOException e) {
throw new UncheckedException(e);
}
}
}

@Override
public void stop() {
super.stop();
if (doProxy && null != proxy) {
try {
proxy.stop();
} finally {
proxy = null;
}
}
}

public String getAccountName() {
return accountName;
}
@@ -234,14 +145,6 @@ public String getAccountKey() {
* carries over.
*/
public int getBlobsPort() {
if (doProxy) {
if (proxy == null) throw new IllegalStateException("");
return proxy.getLocalPort();
}
return getRealBlobsPort();
}

int getRealBlobsPort() {
return super.getMappedPort(blobsPort);
}

This file was deleted.

0 comments on commit ea0fd79

Please sign in to comment.