Skip to content

Commit

Permalink
add old api with storage sdk v8 to prevent BCR
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-mkubik
sfc-gh-mkubik committed Nov 12, 2024
1 parent d045bee commit d117a6b
Show file tree
Hide file tree
Showing 7 changed files with 223 additions and 56 deletions.
27 changes: 24 additions & 3 deletions parent-pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
<awaitility.version>4.2.0</awaitility.version>
<awssdk.version>1.12.655</awssdk.version>
<azure.storage.blob.version>12.26.1</azure.storage.blob.version>
<azure.storage.version>8.0.0</azure.storage.version>
<azuresdk.version>1.2.29</azuresdk.version>
<bouncycastle.version>1.78.1</bouncycastle.version>
<bouncycastle.bcfips.version>1.0.2.5</bouncycastle.bcfips.version>
<bouncycastle.bcpkixfips.version>1.0.7</bouncycastle.bcpkixfips.version>
Expand Down Expand Up @@ -115,6 +117,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-sdk-bom</artifactId>
<version>${azuresdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
Expand Down Expand Up @@ -201,9 +210,9 @@
<version>${google.guava.version}</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>${azure.storage.blob.version}</version>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>${azure.storage.version}</version>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
Expand Down Expand Up @@ -590,6 +599,18 @@
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-common</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
Expand Down
12 changes: 12 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,10 @@
<pattern>com.azure</pattern>
<shadedPattern>${shadeBase}.azure</shadedPattern>
</relocation>
<relocation>
<pattern>com.microsoft.azure</pattern>
<shadedPattern>${shadeBase}.microsoft.azure</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml</pattern>
<shadedPattern>${shadeBase}.fasterxml</shadedPattern>
Expand Down Expand Up @@ -875,6 +879,14 @@
<pattern>io.netty</pattern>
<shadedPattern>${shadeBase}.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>io.projectreactor</pattern>
<shadedPattern>${shadeBase}.io.projectreactor</shadedPattern>
</relocation>
<relocation>
<pattern>org.reactivestreams</pattern>
<shadedPattern>${shadeBase}.org.reactivestreams</shadedPattern>
</relocation>
<relocation>
<pattern>com.carrotsearch</pattern>
<shadedPattern>${shadeBase}.com.carrotsearch</shadedPattern>
Expand Down
68 changes: 68 additions & 0 deletions src/main/java/net/snowflake/client/core/HttpUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
import javax.net.ssl.TrustManager;

import com.microsoft.azure.storage.OperationContext;
import net.snowflake.client.jdbc.ErrorCode;
import net.snowflake.client.jdbc.RestRequest;
import net.snowflake.client.jdbc.SnowflakeDriver;
Expand Down Expand Up @@ -179,6 +181,50 @@ public static void setSessionlessProxyForS3(
S3HttpUtil.setSessionlessProxyForS3(proxyProperties, clientConfig);
}


/**
* A static function to set Azure proxy params for sessionless connections using the proxy params
* from the StageInfo
*
* @param proxyProperties proxy properties
* @param opContext the configuration needed by Azure to set the proxy
* @throws SnowflakeSQLException
*
* @deprecated, please use {@link HttpUtil#setSessionlessProxyForAzure(Properties, HttpClientOptions)} as
* it supports the up-to-date azure storage client.
*/
@Deprecated
public static void setSessionlessProxyForAzure(
Properties proxyProperties, OperationContext opContext) throws SnowflakeSQLException {
if (proxyProperties != null
&& proxyProperties.size() > 0
&& proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey()) != null) {
Boolean useProxy =
Boolean.valueOf(
proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey()));
if (useProxy) {
String proxyHost =
proxyProperties.getProperty(SFSessionProperty.PROXY_HOST.getPropertyKey());
int proxyPort;
try {
proxyPort =
Integer.parseInt(
proxyProperties.getProperty(SFSessionProperty.PROXY_PORT.getPropertyKey()));
} catch (NumberFormatException | NullPointerException e) {
throw new SnowflakeSQLException(
ErrorCode.INVALID_PROXY_PROPERTIES, "Could not parse port number");
}
Proxy azProxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
logger.debug("Setting sessionless Azure proxy. Host: {}, port: {}", proxyHost, proxyPort);
opContext.setProxy(azProxy);
} else {
logger.debug("Omitting sessionless Azure proxy setup as proxy is disabled");
}
} else {
logger.debug("Omitting sessionless Azure proxy setup");
}
}

/**
* A static function to set Azure proxy params for sessionless connections using the proxy params
* from the StageInfo
Expand Down Expand Up @@ -237,6 +283,28 @@ public static void setSessionlessProxyForAzure(
httpClientOptions.setProxyOptions(proxyOptions);
}

/**
* A static function to set Azure proxy params when there is a valid session
*
* @param key key to HttpClient map containing OCSP and proxy info
* @param opContext the configuration needed by Azure to set the proxy
*
* @deprecated Use {@link HttpUtil#setProxyForAzure(HttpClientSettingsKey, OperationContext)} as it supports
* the most recent azure storage client
*/
@Deprecated
public static void setProxyForAzure(HttpClientSettingsKey key, OperationContext opContext) {
if (key != null && key.usesProxy()) {
Proxy azProxy =
new Proxy(Proxy.Type.HTTP, new InetSocketAddress(key.getProxyHost(), key.getProxyPort()));
logger.debug(
"Setting Azure proxy. Host: {}, port: {}", key.getProxyHost(), key.getProxyPort());
opContext.setProxy(azProxy);
} else {
logger.debug("Omitting Azure proxy setup");
}
}

/**
* A static function to set Azure proxy params when there is a valid session
*
Expand Down
36 changes: 33 additions & 3 deletions src/main/java/net/snowflake/client/jdbc/cloud/storage/AzureObjectSummariesIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import java.util.Iterator;
import java.util.NoSuchElementException;

import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.azure.storage.blob.ListBlobItem;
import net.snowflake.client.log.SFLogger;
import net.snowflake.client.log.SFLoggerFactory;

Expand All @@ -24,6 +26,19 @@ public class AzureObjectSummariesIterator implements Iterator<StorageObjectSumma
SFLoggerFactory.getLogger(AzureObjectSummariesIterator.class);
private final String storageLocation;
Iterator<BlobItem> itemIterator;
Iterator<ListBlobItem> listBlobItemIterator;

/*
* Constructs a summaries iterator object from an iterable derived by a
* lostBlobs method
* @param azCloudBlobIterable an iterable set of ListBlobItems
*/
@Deprecated
public AzureObjectSummariesIterator(Iterable<ListBlobItem> azCloudBlobIterable) {
storageLocation = null;
itemIterator = null;
listBlobItemIterator = azCloudBlobIterable.iterator();
}

/*
* Constructs a summaries iterator object from an iterable derived by a
Expand All @@ -33,6 +48,9 @@ public class AzureObjectSummariesIterator implements Iterator<StorageObjectSumma
public AzureObjectSummariesIterator(Iterable<BlobItem> azCloudBlobIterable, String azStorageLocation) {
itemIterator = azCloudBlobIterable.iterator();
storageLocation = azStorageLocation;

// listBlobItem support is deprecated as it comes from azure storage v8
listBlobItemIterator = null;
}

public boolean hasNext() {
Expand All @@ -49,15 +67,27 @@ public boolean hasNext() {
}

public StorageObjectSummary next() {
BlobItem blobItem = itemIterator.next();
if (itemIterator != null) {
BlobItem blobItem = itemIterator.next();
// if (!(blobItem.getProperties().getBlobType() instanceof BlobClient)) {
// // The only other possible type would a CloudDirectory
// // This should never happen since we are listing items as a flat list
// logger.debug("Unexpected listBlobItem instance type: {}", blobItem.getClass());
// throw new IllegalArgumentException("Unexpected listBlobItem instance type");
// }

return StorageObjectSummary.createFromAzureListBlobItem(blobItem, storageLocation);
return StorageObjectSummary.createFromAzureListBlobItem(blobItem, storageLocation);
}
else if (listBlobItemIterator != null) {
ListBlobItem listBlobItem = listBlobItemIterator.next();
if (!(listBlobItem instanceof CloudBlob)) {
// The only other possible type would a CloudDirectory
// This should never happen since we are listing items as a flat list
logger.debug("Unexpected listBlobItem instance type: {}", listBlobItem.getClass());
throw new IllegalArgumentException("Unexpected listBlobItem instance type");
}
return StorageObjectSummary.createFromAzureListBlobItem(listBlobItem);
}
throw new RuntimeException("No azure blob iterator was initialized, should never happen");
}

public void remove() {
Expand Down
51 changes: 51 additions & 0 deletions src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,16 @@
import com.azure.storage.blob.models.BlobStorageException;
import com.google.cloud.storage.Blob;

import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.BlobProperties;
import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.ListBlobItem;
import net.snowflake.client.log.SFLogger;
import net.snowflake.client.log.SFLoggerFactory;

import java.net.URISyntaxException;

/**
* Storage platform-agnostic class that encapsulates remote storage object properties
*
Expand Down Expand Up @@ -57,6 +64,50 @@ public static StorageObjectSummary createFromS3ObjectSummary(S3ObjectSummary obj
objSummary.getSize());
}

/**
* Constructs a StorageObjectSummary object from Azure BLOB properties Using factory methods to
* create these objects since Azure can throw, while retrieving the BLOB properties
*
* @param listBlobItem an Azure ListBlobItem object
* @return the ObjectSummary object created
*/
@Deprecated
public static StorageObjectSummary createFromAzureListBlobItem(ListBlobItem listBlobItem)
throws StorageProviderException {
String location, key, md5;
long size;
CloudBlobContainer container;

// Retrieve the BLOB properties that we need for the Summary
// Azure Storage stores metadata inside each BLOB, therefore the listBlobItem
// will point us to the underlying BLOB and will get the properties from it
// During the process the Storage Client could fail, hence we need to wrap the
// get calls in try/catch and handle possible exceptions
try {
container = listBlobItem.getContainer();
location = container.getName();
CloudBlob cloudBlob = (CloudBlob) listBlobItem;
key = cloudBlob.getName();
BlobProperties blobProperties = cloudBlob.getProperties();
// the content md5 property is not always the actual md5 of the file. But for here, it's only
// used for skipping file on PUT command, hence is ok.
md5 = convertBase64ToHex(blobProperties.getContentMD5().getBytes());
size = blobProperties.getLength();
} catch (URISyntaxException | StorageException ex) {
// This should only happen if somehow we got here with and invalid URI (it should never
// happen)
// ...or there is a Storage service error. Unlike S3, Azure fetches metadata from the BLOB
// itself,
// and its a lazy operation
logger.debug("Failed to create StorageObjectSummary from Azure ListBlobItem: {}", ex);
throw new StorageProviderException(ex);
} catch (Throwable th) {
logger.debug("Failed to create StorageObjectSummary from Azure ListBlobItem: {}", th);
throw th;
}
return new StorageObjectSummary(location, key, md5, size);
}

/**
* Constructs a StorageObjectSummary object from Azure BLOB properties Using factory methods to
* create these objects since Azure can throw, while retrieving the BLOB properties
Expand Down
19 changes: 15 additions & 4 deletions src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageObjectSummaryCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import com.azure.storage.blob.models.BlobItem;
import com.google.api.gax.paging.Page;
import com.google.cloud.storage.Blob;
import com.microsoft.azure.storage.blob.ListBlobItem;

import java.util.Iterator;
import java.util.List;

Expand All @@ -27,7 +29,8 @@ private enum storageType {

private final storageType sType;
private List<S3ObjectSummary> s3ObjSummariesList = null;
private Iterable<BlobItem> azCLoudBlobIterable = null;
private Iterable<ListBlobItem> azCLoudBlobIterable = null;
private Iterable<BlobItem> azCloudBlobItemIterable = null;
private Page<Blob> gcsIterablePage = null;

// Constructs platform-agnostic collection of object summaries from S3 object summaries
Expand All @@ -38,12 +41,17 @@ public StorageObjectSummaryCollection(List<S3ObjectSummary> s3ObjectSummaries) {

// Constructs platform-agnostic collection of object summaries from an Azure CloudBlobDirectory
// object
public StorageObjectSummaryCollection(Iterable<BlobItem> azCLoudBlobIterable, String remoteStorageLocation) {
this.azCLoudBlobIterable = azCLoudBlobIterable;
public StorageObjectSummaryCollection(Iterable<BlobItem> azCLoudBlobItemIterable, String remoteStorageLocation) {
this.azCloudBlobItemIterable = azCLoudBlobItemIterable;
this.azStorageLocation = remoteStorageLocation;
sType = storageType.AZURE;
}

public StorageObjectSummaryCollection(Iterable<ListBlobItem> azCLoudBlobIterable) {
this.azCLoudBlobIterable = azCLoudBlobIterable;
sType = storageType.AZURE;
}

public StorageObjectSummaryCollection(Page<Blob> gcsIterablePage) {
this.gcsIterablePage = gcsIterablePage;
sType = storageType.GCS;
Expand All @@ -56,9 +64,12 @@ public Iterator<StorageObjectSummary> iterator() {
return new S3ObjectSummariesIterator(s3ObjSummariesList);
case AZURE:
if (azStorageLocation == null) {
if (azCLoudBlobIterable != null) {
return new AzureObjectSummariesIterator(azCLoudBlobIterable);
}
throw new RuntimeException("Storage type is Azure but azStorageLocation field is not set. Should never happen");
}
return new AzureObjectSummariesIterator(azCLoudBlobIterable, azStorageLocation);
return new AzureObjectSummariesIterator(azCloudBlobItemIterable, azStorageLocation);
case GCS:
return new GcsObjectSummariesIterator(this.gcsIterablePage);
default:
Expand Down
Loading

0 comments on commit d117a6b

Please sign in to comment.