diff --git a/parent-pom.xml b/parent-pom.xml index 5b19171e5..6c9c3705c 100644 --- a/parent-pom.xml +++ b/parent-pom.xml @@ -26,6 +26,8 @@ 4.2.0 1.12.655 12.26.1 + 8.0.0 + 1.2.29 1.78.1 1.0.2.5 1.0.7 @@ -115,6 +117,13 @@ pom import + + com.azure + azure-sdk-bom + ${azuresdk.version} + pom + import + com.fasterxml.jackson jackson-bom @@ -201,9 +210,9 @@ ${google.guava.version} - com.azure - azure-storage-blob - ${azure.storage.blob.version} + com.microsoft.azure + azure-storage + ${azure.storage.version} com.nimbusds @@ -590,6 +599,18 @@ com.azure azure-storage-blob + + com.azure + azure-core + + + com.azure + azure-storage-common + + + com.microsoft.azure + azure-storage + com.nimbusds nimbus-jose-jwt diff --git a/pom.xml b/pom.xml index 1776dc0e8..5dfbf8def 100644 --- a/pom.xml +++ b/pom.xml @@ -815,6 +815,10 @@ com.azure ${shadeBase}.azure + + com.microsoft.azure + ${shadeBase}.microsoft.azure + com.fasterxml ${shadeBase}.fasterxml @@ -875,6 +879,14 @@ io.netty ${shadeBase}.io.netty + + io.projectreactor + ${shadeBase}.io.projectreactor + + + org.reactivestreams + ${shadeBase}.org.reactivestreams + com.carrotsearch ${shadeBase}.com.carrotsearch diff --git a/src/main/java/net/snowflake/client/core/HttpUtil.java b/src/main/java/net/snowflake/client/core/HttpUtil.java index 8b5f320e2..eaa5247b6 100644 --- a/src/main/java/net/snowflake/client/core/HttpUtil.java +++ b/src/main/java/net/snowflake/client/core/HttpUtil.java @@ -31,6 +31,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; import javax.net.ssl.TrustManager; + +import com.microsoft.azure.storage.OperationContext; import net.snowflake.client.jdbc.ErrorCode; import net.snowflake.client.jdbc.RestRequest; import net.snowflake.client.jdbc.SnowflakeDriver; @@ -179,6 +181,50 @@ public static void setSessionlessProxyForS3( S3HttpUtil.setSessionlessProxyForS3(proxyProperties, clientConfig); } + + /** + * A static function to set Azure proxy params for sessionless connections using the proxy params + * from the StageInfo + * + * @param proxyProperties proxy properties + * @param opContext the configuration needed by Azure to set the proxy + * @throws SnowflakeSQLException + * + * @deprecated, please use {@link HttpUtil#setSessionlessProxyForAzure(Properties, HttpClientOptions)} as + * it supports the up-to-date azure storage client. + */ + @Deprecated + public static void setSessionlessProxyForAzure( + Properties proxyProperties, OperationContext opContext) throws SnowflakeSQLException { + if (proxyProperties != null + && proxyProperties.size() > 0 + && proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey()) != null) { + Boolean useProxy = + Boolean.valueOf( + proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey())); + if (useProxy) { + String proxyHost = + proxyProperties.getProperty(SFSessionProperty.PROXY_HOST.getPropertyKey()); + int proxyPort; + try { + proxyPort = + Integer.parseInt( + proxyProperties.getProperty(SFSessionProperty.PROXY_PORT.getPropertyKey())); + } catch (NumberFormatException | NullPointerException e) { + throw new SnowflakeSQLException( + ErrorCode.INVALID_PROXY_PROPERTIES, "Could not parse port number"); + } + Proxy azProxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + logger.debug("Setting sessionless Azure proxy. Host: {}, port: {}", proxyHost, proxyPort); + opContext.setProxy(azProxy); + } else { + logger.debug("Omitting sessionless Azure proxy setup as proxy is disabled"); + } + } else { + logger.debug("Omitting sessionless Azure proxy setup"); + } + } + /** * A static function to set Azure proxy params for sessionless connections using the proxy params * from the StageInfo @@ -237,6 +283,28 @@ public static void setSessionlessProxyForAzure( httpClientOptions.setProxyOptions(proxyOptions); } + /** + * A static function to set Azure proxy params when there is a valid session + * + * @param key key to HttpClient map containing OCSP and proxy info + * @param opContext the configuration needed by Azure to set the proxy + * + * @deprecated Use {@link HttpUtil#setProxyForAzure(HttpClientSettingsKey, OperationContext)} as it supports + * the most recent azure storage client + */ + @Deprecated + public static void setProxyForAzure(HttpClientSettingsKey key, OperationContext opContext) { + if (key != null && key.usesProxy()) { + Proxy azProxy = + new Proxy(Proxy.Type.HTTP, new InetSocketAddress(key.getProxyHost(), key.getProxyPort())); + logger.debug( + "Setting Azure proxy. Host: {}, port: {}", key.getProxyHost(), key.getProxyPort()); + opContext.setProxy(azProxy); + } else { + logger.debug("Omitting Azure proxy setup"); + } + } + /** * A static function to set Azure proxy params when there is a valid session * diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/AzureObjectSummariesIterator.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/AzureObjectSummariesIterator.java index 3f4b02729..7f284359c 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/AzureObjectSummariesIterator.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/AzureObjectSummariesIterator.java @@ -10,6 +10,8 @@ import java.util.Iterator; import java.util.NoSuchElementException; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.ListBlobItem; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; @@ -24,6 +26,19 @@ public class AzureObjectSummariesIterator implements Iterator itemIterator; + Iterator listBlobItemIterator; + + /* + * Constructs a summaries iterator object from an iterable derived by a + * lostBlobs method + * @param azCloudBlobIterable an iterable set of ListBlobItems + */ + @Deprecated + public AzureObjectSummariesIterator(Iterable azCloudBlobIterable) { + storageLocation = null; + itemIterator = null; + listBlobItemIterator = azCloudBlobIterable.iterator(); + } /* * Constructs a summaries iterator object from an iterable derived by a @@ -33,6 +48,9 @@ public class AzureObjectSummariesIterator implements Iterator azCloudBlobIterable, String azStorageLocation) { itemIterator = azCloudBlobIterable.iterator(); storageLocation = azStorageLocation; + + // listBlobItem support is deprecated as it comes from azure storage v8 + listBlobItemIterator = null; } public boolean hasNext() { @@ -49,15 +67,27 @@ public boolean hasNext() { } public StorageObjectSummary next() { - BlobItem blobItem = itemIterator.next(); + if (itemIterator != null) { + BlobItem blobItem = itemIterator.next(); // if (!(blobItem.getProperties().getBlobType() instanceof BlobClient)) { // // The only other possible type would a CloudDirectory // // This should never happen since we are listing items as a flat list // logger.debug("Unexpected listBlobItem instance type: {}", blobItem.getClass()); // throw new IllegalArgumentException("Unexpected listBlobItem instance type"); // } - - return StorageObjectSummary.createFromAzureListBlobItem(blobItem, storageLocation); + return StorageObjectSummary.createFromAzureListBlobItem(blobItem, storageLocation); + } + else if (listBlobItemIterator != null) { + ListBlobItem listBlobItem = listBlobItemIterator.next(); + if (!(listBlobItem instanceof CloudBlob)) { + // The only other possible type would a CloudDirectory + // This should never happen since we are listing items as a flat list + logger.debug("Unexpected listBlobItem instance type: {}", listBlobItem.getClass()); + throw new IllegalArgumentException("Unexpected listBlobItem instance type"); + } + return StorageObjectSummary.createFromAzureListBlobItem(listBlobItem); + } + throw new RuntimeException("No azure blob iterator was initialized, should never happen"); } public void remove() { diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummary.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummary.java index 5dc82db5f..d672f7061 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummary.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummary.java @@ -9,9 +9,16 @@ import com.azure.storage.blob.models.BlobStorageException; import com.google.cloud.storage.Blob; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.ListBlobItem; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; +import java.net.URISyntaxException; + /** * Storage platform-agnostic class that encapsulates remote storage object properties * @@ -57,6 +64,50 @@ public static StorageObjectSummary createFromS3ObjectSummary(S3ObjectSummary obj objSummary.getSize()); } + /** + * Constructs a StorageObjectSummary object from Azure BLOB properties Using factory methods to + * create these objects since Azure can throw, while retrieving the BLOB properties + * + * @param listBlobItem an Azure ListBlobItem object + * @return the ObjectSummary object created + */ + @Deprecated + public static StorageObjectSummary createFromAzureListBlobItem(ListBlobItem listBlobItem) + throws StorageProviderException { + String location, key, md5; + long size; + CloudBlobContainer container; + + // Retrieve the BLOB properties that we need for the Summary + // Azure Storage stores metadata inside each BLOB, therefore the listBlobItem + // will point us to the underlying BLOB and will get the properties from it + // During the process the Storage Client could fail, hence we need to wrap the + // get calls in try/catch and handle possible exceptions + try { + container = listBlobItem.getContainer(); + location = container.getName(); + CloudBlob cloudBlob = (CloudBlob) listBlobItem; + key = cloudBlob.getName(); + BlobProperties blobProperties = cloudBlob.getProperties(); + // the content md5 property is not always the actual md5 of the file. But for here, it's only + // used for skipping file on PUT command, hence is ok. + md5 = convertBase64ToHex(blobProperties.getContentMD5().getBytes()); + size = blobProperties.getLength(); + } catch (URISyntaxException | StorageException ex) { + // This should only happen if somehow we got here with and invalid URI (it should never + // happen) + // ...or there is a Storage service error. Unlike S3, Azure fetches metadata from the BLOB + // itself, + // and its a lazy operation + logger.debug("Failed to create StorageObjectSummary from Azure ListBlobItem: {}", ex); + throw new StorageProviderException(ex); + } catch (Throwable th) { + logger.debug("Failed to create StorageObjectSummary from Azure ListBlobItem: {}", th); + throw th; + } + return new StorageObjectSummary(location, key, md5, size); + } + /** * Constructs a StorageObjectSummary object from Azure BLOB properties Using factory methods to * create these objects since Azure can throw, while retrieving the BLOB properties diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummaryCollection.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummaryCollection.java index 4460988b0..4a454332b 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummaryCollection.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummaryCollection.java @@ -7,6 +7,8 @@ import com.azure.storage.blob.models.BlobItem; import com.google.api.gax.paging.Page; import com.google.cloud.storage.Blob; +import com.microsoft.azure.storage.blob.ListBlobItem; + import java.util.Iterator; import java.util.List; @@ -27,7 +29,8 @@ private enum storageType { private final storageType sType; private List s3ObjSummariesList = null; - private Iterable azCLoudBlobIterable = null; + private Iterable azCLoudBlobIterable = null; + private Iterable azCloudBlobItemIterable = null; private Page gcsIterablePage = null; // Constructs platform-agnostic collection of object summaries from S3 object summaries @@ -38,12 +41,17 @@ public StorageObjectSummaryCollection(List s3ObjectSummaries) { // Constructs platform-agnostic collection of object summaries from an Azure CloudBlobDirectory // object - public StorageObjectSummaryCollection(Iterable azCLoudBlobIterable, String remoteStorageLocation) { - this.azCLoudBlobIterable = azCLoudBlobIterable; + public StorageObjectSummaryCollection(Iterable azCLoudBlobItemIterable, String remoteStorageLocation) { + this.azCloudBlobItemIterable = azCLoudBlobItemIterable; this.azStorageLocation = remoteStorageLocation; sType = storageType.AZURE; } + public StorageObjectSummaryCollection(Iterable azCLoudBlobIterable) { + this.azCLoudBlobIterable = azCLoudBlobIterable; + sType = storageType.AZURE; + } + public StorageObjectSummaryCollection(Page gcsIterablePage) { this.gcsIterablePage = gcsIterablePage; sType = storageType.GCS; @@ -56,9 +64,12 @@ public Iterator iterator() { return new S3ObjectSummariesIterator(s3ObjSummariesList); case AZURE: if (azStorageLocation == null) { + if (azCLoudBlobIterable != null) { + return new AzureObjectSummariesIterator(azCLoudBlobIterable); + } throw new RuntimeException("Storage type is Azure but azStorageLocation field is not set. Should never happen"); } - return new AzureObjectSummariesIterator(azCLoudBlobIterable, azStorageLocation); + return new AzureObjectSummariesIterator(azCloudBlobItemIterable, azStorageLocation); case GCS: return new GcsObjectSummariesIterator(this.gcsIterablePage); default: diff --git a/src/test/java/net/snowflake/client/jdbc/SnowflakeAzureClientHandleExceptionLatestIT.java b/src/test/java/net/snowflake/client/jdbc/SnowflakeAzureClientHandleExceptionLatestIT.java index 251042d9b..8827f1bb8 100644 --- a/src/test/java/net/snowflake/client/jdbc/SnowflakeAzureClientHandleExceptionLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/SnowflakeAzureClientHandleExceptionLatestIT.java @@ -6,8 +6,6 @@ import java.io.File; import java.io.IOException; import java.net.SocketTimeoutException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.security.InvalidKeyException; import java.sql.Connection; import java.sql.SQLException; @@ -32,12 +30,16 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.Mockito; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; /** Test for SnowflakeAzureClient handle exception function */ @Category(TestCategoryOthers.class) +@RunWith(MockitoJUnitRunner.class) public class SnowflakeAzureClientHandleExceptionLatestIT extends AbstractDriverIT { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); private Connection connection; @@ -47,6 +49,13 @@ public class SnowflakeAzureClientHandleExceptionLatestIT extends AbstractDriverI private SnowflakeAzureClient spyingClient; private int overMaxRetry; private int maxRetry; + @Mock + HttpResponse mockHttpResponse; + + /** + * @see https://learn.microsoft.com/en-us/rest/api/storageservices/status-and-error-codes2 + */ + private final static String AZURE_ERROR_CODE_HEADER = "x-ms-error-code"; @Before public void setup() throws SQLException { @@ -64,45 +73,10 @@ public void setup() throws SQLException { maxRetry = client.getMaxRetries(); overMaxRetry = maxRetry + 1; spyingClient = Mockito.spy(client); - } - private HttpResponse constructHttpResponse(int httpStatus) { - return new HttpResponse(null) { - @Override - public int getStatusCode() { - return httpStatus; - } - - @Override - public String getHeaderValue(String s) { - return ""; - } - - @Override - public HttpHeaders getHeaders() { - return null; - } - - @Override - public Flux getBody() { - return null; - } - - @Override - public Mono getBodyAsByteArray() { - return null; - } - - @Override - public Mono getBodyAsString() { - return null; - } - - @Override - public Mono getBodyAsString(Charset charset) { - return null; - } - }; + when(mockHttpResponse.getStatusCode()).thenReturn(HttpStatus.SC_FORBIDDEN); + when(mockHttpResponse.getHeaders()).thenReturn(new HttpHeaders() + .add(AZURE_ERROR_CODE_HEADER, "InvalidAuthenticationInfo")); } @Test @@ -111,7 +85,7 @@ public void error403RenewExpired() throws SQLException, InterruptedException { // Unauthenticated, renew is called. spyingClient.handleStorageException( new BlobStorageException( - "Unauthenticated", constructHttpResponse(HttpStatus.SC_FORBIDDEN), new Exception()), + "Unauthenticated", mockHttpResponse, new Exception()), 0, "upload", sfSession, @@ -130,7 +104,7 @@ public void run() { spyingClient.handleStorageException( new BlobStorageException( "Unauthenticated", - constructHttpResponse(HttpStatus.SC_FORBIDDEN), + mockHttpResponse, new Exception()), maxRetry, "upload", @@ -154,7 +128,7 @@ public void run() { public void error403OverMaxRetryThrow() throws SQLException { spyingClient.handleStorageException( new BlobStorageException( - "Unauthenticated", constructHttpResponse(HttpStatus.SC_FORBIDDEN), new Exception()), + "Unauthenticated", mockHttpResponse, new Exception()), overMaxRetry, "upload", sfSession, @@ -167,7 +141,7 @@ public void error403OverMaxRetryThrow() throws SQLException { public void error403NullSession() throws SQLException { spyingClient.handleStorageException( new BlobStorageException( - "Unauthenticated", constructHttpResponse(HttpStatus.SC_FORBIDDEN), new Exception()), + "Unauthenticated", mockHttpResponse, new Exception()), 0, "upload", null,