fix(writer): Initialise retry settings with a delay between retires (G…

…oogleCloudDataproc#1030)

- The current retry settings was initialised with max attempts only, this initialises the delay between runs to 0, meaning that the 5 calls will happen with 0 millis between them and not actually an exponential retry strategy like it implies here: https://github.com/GoogleCloudDataproc/spark-bigquery-connector/blob/741bdb33bf5c06fb3fad9defffa65ea1470d2690/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryDirectDataWriterHelper.java#L133-L142 (ran a test for the retryCallable method with the current settings and indeed the delay was 0 between calls)
- So we initialise, it with the same settings as reading (can be tuned later) as otherwise the API is hammered leading to all failures

spark-bigquery-connector-common/src/main/java/com/google/cloud/spark/bigquery/SparkBigQueryConfig.java

@@ -211,9 +211,6 @@ public static WriteMethod from(@Nullable String writeMethod) {
   private CompressionCodec arrowCompressionCodec = DEFAULT_ARROW_COMPRESSION_CODEC;
   private WriteMethod writeMethod = DEFAULT_WRITE_METHOD;
   boolean writeAtLeastOnce = false;
-  // for V2 write with BigQuery Storage Write API
-  RetrySettings bigqueryDataWriteHelperRetrySettings =
-      RetrySettings.newBuilder().setMaxAttempts(5).build();
   private int cacheExpirationTimeInMinutes = DEFAULT_CACHE_EXPIRATION_IN_MINUTES;
   // used to create BigQuery ReadSessions
   private com.google.common.base.Optional<String> traceId;
@@ -947,11 +944,14 @@ public Optional<String> getKmsKeyName() {
 
   @Override
   public RetrySettings getBigQueryClientRetrySettings() {
+    int maxAttempts =
+        sparkBigQueryProxyAndHttpConfig.getHttpMaxRetry().orElse(DEFAULT_BIGQUERY_CLIENT_RETRIES);
+    return getRetrySettings(maxAttempts);
+  }
+
+  private static RetrySettings getRetrySettings(int maxAttempts) {
     return RetrySettings.newBuilder()
-        .setMaxAttempts(
-            sparkBigQueryProxyAndHttpConfig
-                .getHttpMaxRetry()
-                .orElse(DEFAULT_BIGQUERY_CLIENT_RETRIES))
+        .setMaxAttempts(maxAttempts)
         .setTotalTimeout(Duration.ofMinutes(10))
         .setInitialRpcTimeout(Duration.ofSeconds(60))
         .setMaxRpcTimeout(Duration.ofMinutes(5))
@@ -962,8 +962,9 @@ public RetrySettings getBigQueryClientRetrySettings() {
         .build();
   }
 
+  // for V2 write with BigQuery Storage Write API
   public RetrySettings getBigqueryDataWriteHelperRetrySettings() {
-    return bigqueryDataWriteHelperRetrySettings;
+    return getRetrySettings(5);
   }
 
   public WriteMethod getWriteMethod() {