From 4be13fef99bff7619e6a2334cc8073d83a6afacf Mon Sep 17 00:00:00 2001
From: "randy.pierce" <randy.pierce@noaa.gov>
Date: Wed, 14 Feb 2024 09:48:30 -0700
Subject: [PATCH 1/2] add retries to other connections

---
 src/vxingest/builder_common/ingest_manager.py |  3 ++-
 src/vxingest/builder_common/vx_ingest.py      | 18 ++++++++++++++---
 src/vxingest/main.py                          | 20 +++++++++++++++----
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/vxingest/builder_common/ingest_manager.py b/src/vxingest/builder_common/ingest_manager.py
index 6ce5a37..882da60 100644
--- a/src/vxingest/builder_common/ingest_manager.py
+++ b/src/vxingest/builder_common/ingest_manager.py
@@ -106,7 +106,8 @@ def connect_cb(self):
 
         try:
             timeout_options = ClusterTimeoutOptions(
-                kv_timeout=timedelta(seconds=25), query_timeout=timedelta(seconds=120)
+                kv_timeout=timedelta(seconds=25),
+                query_timeout=timedelta(seconds=120),
             )
             options = ClusterOptions(
                 PasswordAuthenticator(
diff --git a/src/vxingest/builder_common/vx_ingest.py b/src/vxingest/builder_common/vx_ingest.py
index 392a1f3..00fbbad 100644
--- a/src/vxingest/builder_common/vx_ingest.py
+++ b/src/vxingest/builder_common/vx_ingest.py
@@ -31,6 +31,7 @@
 import yaml
 from couchbase.auth import PasswordAuthenticator
 from couchbase.cluster import Cluster
+from couchbase.exceptions import CouchbaseException
 from couchbase.options import ClusterOptions, ClusterTimeoutOptions
 
 # Get a logger with this module's name to help with debugging
@@ -144,9 +145,20 @@ def connect_cb(self):
                 ),
                 timeout_options=timeout_options,
             )
-            self.cluster = Cluster(
-                "couchbase://" + self.cb_credentials["host"], options
-            )
+            _attempts = 0
+            while _attempts < 3:
+                try:
+                    self.cluster = Cluster(
+                        "couchbase://" + self.cb_credentials["host"], options
+                    )
+                    break
+                except CouchbaseException as _e:
+                    time.sleep(5)
+                    _attempts = _attempts + 1
+            if _attempts == 3:
+                raise CouchbaseException(
+                    "Could not connect to couchbase after 3 attempts"
+                )
             self.collection = self.cluster.bucket(
                 self.cb_credentials["bucket"]
             ).collection(self.cb_credentials["collection"])
diff --git a/src/vxingest/main.py b/src/vxingest/main.py
index 91b8f96..20a1692 100644
--- a/src/vxingest/main.py
+++ b/src/vxingest/main.py
@@ -7,6 +7,7 @@
 import shutil
 import sys
 import tarfile
+import time
 from datetime import datetime, timedelta
 from multiprocessing import Queue, set_start_method
 from pathlib import Path
@@ -285,10 +286,21 @@ def connect_cb(creds: dict[str, str]) -> Cluster:
     # Get a reference to our cluster
     # NOTE: For TLS/SSL connection use 'couchbases://<your-ip-address>' instead
     logger.info(f"Connecting to Couchbase at: {creds['cb_host']}")
-    cluster = Cluster(
-        f"couchbase://{creds['cb_host']}",
-        ClusterOptions(auth, timeout_options=timeout_config),  # type: ignore
-    )
+    _attempts = 0
+    while _attempts < 3:
+        try:
+            cluster = Cluster(
+                f"couchbase://{creds['cb_host']}",
+                ClusterOptions(auth, timeout_options=timeout_config),  # type: ignore
+            )
+            break
+        except CouchbaseException as _e:
+            time.sleep(5)
+            _attempts = _attempts + 1
+    if _attempts == 3:
+        raise CouchbaseException(
+            "Could not connect to couchbase after 3 attempts"
+        )
 
     # Wait until the cluster is ready for use.
     cluster.wait_until_ready(timedelta(seconds=5))

From 892e36645762ce16d3cc5fb365a7a487175b7ce9 Mon Sep 17 00:00:00 2001
From: "randy.pierce" <randy.pierce@noaa.gov>
Date: Wed, 14 Feb 2024 10:08:47 -0700
Subject: [PATCH 2/2] format

---
 src/vxingest/main.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/vxingest/main.py b/src/vxingest/main.py
index 20a1692..ac2f96a 100644
--- a/src/vxingest/main.py
+++ b/src/vxingest/main.py
@@ -298,9 +298,7 @@ def connect_cb(creds: dict[str, str]) -> Cluster:
             time.sleep(5)
             _attempts = _attempts + 1
     if _attempts == 3:
-        raise CouchbaseException(
-            "Could not connect to couchbase after 3 attempts"
-        )
+        raise CouchbaseException("Could not connect to couchbase after 3 attempts")
 
     # Wait until the cluster is ready for use.
     cluster.wait_until_ready(timedelta(seconds=5))