make tests work with new indexes and new credentials

.gitignore

@@ -3,11 +3,11 @@
 .fleet/
 .vscode/
 .history/
-
 # Mac
 .DS_Store
 
 # Project-specific
+.env*
 [Ll]ogs/
 test_venv/
 grib2_to_cb_test/test/2119312000003

README.md

@@ -46,6 +46,8 @@ cb_scope: "_default"
 cb_collection: "METAR"
 ```
 
+The cb_host file requires a protocol. For example ... "couchbase://adb-cb1.gsd.esrl.noaa.gov" - because adb-cb1... is a single node cluster. For adb-cb2 (which is one node of a multinode cluster) it would be "couchbases://adb-cb2.gsd.esrl.noaa.gov". Any of the nodes would suffice.
+
 Once that's in place, you can run the ingest with Docker Compose like the example below. Note the `public` and `data` env variables respectively point to where the input data resides and where you'd like the container to write out to. They are the only part of the command you would need to modify.
 
 ```bash

src/vxingest/builder_common/ingest_manager.py

@@ -118,9 +118,7 @@ def connect_cb(self):
             _attempts = 0
             while _attempts < 3:
                 try:
-                    self.cluster = Cluster(
-                        "couchbase://" + self.cb_credentials["host"], options
-                    )
+                    self.cluster = Cluster(self.cb_credentials["host"], options)
                     break
                 except CouchbaseException as _e:
                     time.sleep(5)

src/vxingest/builder_common/load_backup_ingest_docs.py

@@ -101,9 +101,7 @@ def connect_cb(self):
                     self.cb_credentials["user"], self.cb_credentials["password"]
                 )
             )
-            self.cluster = Cluster(
-                "couchbase://" + self.cb_credentials["host"], options
-            )
+            self.cluster = Cluster(self.cb_credentials["host"], options)
             self.collection = self.cluster.bucket("mdata").default_collection()
         except Exception as e:
             print(f"*** Error in connect_cb *** {e}")

src/vxingest/builder_common/vx_ingest.py

@@ -148,9 +148,7 @@ def connect_cb(self):
             _attempts = 0
             while _attempts < 3:
                 try:
-                    self.cluster = Cluster(
-                        "couchbase://" + self.cb_credentials["host"], options
-                    )
+                    self.cluster = Cluster(self.cb_credentials["host"], options)
                     break
                 except CouchbaseException as _e:
                     time.sleep(5)

src/vxingest/utilities/backfill_obs_with_rh.py

@@ -40,7 +40,7 @@ def setup_connection():
         )
         options = ClusterOptions(PasswordAuthenticator(_user, _password))
         connection = {}
-        connection["cluster"] = Cluster("couchbase://" + _host, options)
+        connection["cluster"] = Cluster(_host, options)
         connection["bucket"] = connection["cluster"].bucket(_bucket)
         connection["scope"] = connection["bucket"].scope(_scope)
         connection["collection"] = connection["scope"].collection(_collection)

tests/vxingest/builder_common/test_unit_queries.py

@@ -34,7 +34,7 @@ def connect_cb():
         PasswordAuthenticator(cb_connection["user"], cb_connection["password"]),
         timeout_options=timeout_options,
     )
-    cb_connection["cluster"] = Cluster("couchbase://" + cb_connection["host"], options)
+    cb_connection["cluster"] = Cluster(cb_connection["host"], options)
     cb_connection["collection"] = (
         cb_connection["cluster"]
         .bucket(cb_connection["bucket"])
@@ -65,7 +65,7 @@ def test_stations_fcst_valid_epoch(request):
 
 @pytest.mark.integration()
 def test_stations_get_file_list_grib2(request):
-    _expected_time = 10
+    _expected_time = 16
     _name = request.node.name
     testdata = Path("tests/vxingest/builder_common/testdata/get_file_list_grib2.n1ql")
     with testdata.open(mode="r", encoding="utf-8") as file:

tests/vxingest/ctc_to_cb/test_int_metar_ctc.py

@@ -58,9 +58,9 @@ def test_check_fcst_valid_epoch_fcst_valid_iso():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     options = ClusterOptions(PasswordAuthenticator(_user, _password))
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     stmnt = f"""SELECT m0.fcstValidEpoch fve, fcstValidISO fvi
         FROM `{_bucket}`.{_scope}.{_collection} m0
         WHERE
@@ -105,7 +105,7 @@ def test_get_stations_geo_search():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     collection = cluster.bucket(_bucket).scope(_scope).collection(_collection)
     load_spec = {}
     load_spec["cluster"] = cluster
@@ -199,7 +199,7 @@ def calculate_cb_ctc(
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     collection = cluster.bucket(_bucket).scope(_scope).collection(_collection)
     load_spec = {}
     load_spec["cluster"] = cluster
@@ -498,7 +498,7 @@ def test_ctc_ceiling_data_hrrr_ops_all_hrrr():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     # get available fcstValidEpochs for couchbase
 
     result = cluster.query(
@@ -631,7 +631,7 @@ def test_ctc_visibiltiy_data_hrrr_ops_all_hrrr():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     # get available fcstValidEpochs for couchbase
 
     stmnt = f"""SELECT RAW fcstValidEpoch

tests/vxingest/ctc_to_cb/test_unit_queries_ctc.py

@@ -35,7 +35,7 @@ def connect_cb():
         PasswordAuthenticator(cb_connection["user"], cb_connection["password"]),
         timeout_options=timeout_options,
     )
-    cb_connection["cluster"] = Cluster("couchbase://" + cb_connection["host"], options)
+    cb_connection["cluster"] = Cluster(cb_connection["host"], options)
     cb_connection["collection"] = (
         cb_connection["cluster"]
         .bucket(cb_connection["bucket"])
@@ -137,7 +137,7 @@ def test_get_stations(request):
 @pytest.mark.integration()
 def test_get_threshold_descriptions(request):
     _name = request.node.name
-    _expected_time = 0.01
+    _expected_time = 0.6
     testdata = Path(
         "tests/vxingest/ctc_to_cb/testdata/test_get_threshold_descriptions.n1ql"
     )

tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py

@@ -54,9 +54,7 @@ def connect_cb():
             PasswordAuthenticator(cb_connection["user"], cb_connection["password"]),
             timeout_options=timeout_options,
         )
-        cb_connection["cluster"] = Cluster(
-            "couchbase://" + cb_connection["host"], options
-        )
+        cb_connection["cluster"] = Cluster(cb_connection["host"], options)
         cb_connection["collection"] = (
             cb_connection["cluster"]
             .bucket(cb_connection["bucket"])

tests/vxingest/grib2_to_cb/test_unit_queries_grib.py

@@ -35,7 +35,7 @@ def connect_cb():
         PasswordAuthenticator(cb_connection["user"], cb_connection["password"]),
         timeout_options=timeout_options,
     )
-    cb_connection["cluster"] = Cluster("couchbase://" + cb_connection["host"], options)
+    cb_connection["cluster"] = Cluster(cb_connection["host"], options)
     cb_connection["collection"] = (
         cb_connection["cluster"]
         .bucket(cb_connection["bucket"])

tests/vxingest/grib2_to_cb/testdata/test_ingest_document_fields.n1ql

@@ -1,4 +1,4 @@
 Select file_mask,
     input_data_path
 from `vxdata`._default.METAR
-where meta().id = "MD:V01:METAR:RAP_OPS_130:ingest:grib2";
+USE KEYS ["MD:V01:METAR:RAP_OPS_130:ingest:grib2"];

tests/vxingest/grib2_to_cb/testdata/test_ingest_document_id.n1ql

@@ -1,3 +1,3 @@
 Select ingest_document_ids
 from `vxdata`._default.METAR
-where meta().id = "MD:V01:METAR:RAP_OPS_130:ingest:grib2";
+USE KEYS ["MD:V01:METAR:RAP_OPS_130:ingest:grib2"];

tests/vxingest/netcdf_to_cb/test_unit_queries_obs.py

@@ -34,7 +34,7 @@ def connect_cb():
         PasswordAuthenticator(cb_connection["user"], cb_connection["password"]),
         timeout_options=timeout_options,
     )
-    cb_connection["cluster"] = Cluster("couchbase://" + cb_connection["host"], options)
+    cb_connection["cluster"] = Cluster(cb_connection["host"], options)
     cb_connection["collection"] = (
         cb_connection["cluster"]
         .bucket(cb_connection["bucket"])

tests/vxingest/netcdf_to_cb/testdata/test_get_stations.n1ql

@@ -1,3 +1,3 @@
 Select ingest_document_ids
 from `vxdata`._default.METAR
-where meta().id = "MD:V01:METAR:obs:ingest:netcdf";
+USE KEYS ["MD:V01:METAR:obs:ingest:netcdf"];

tests/vxingest/netcdf_to_cb/testdata/test_ingest_document_fields.n1ql

@@ -1,4 +1,4 @@
 Select file_mask,
     input_data_path
 from `vxdata`._default.METAR
-where meta().id = "MD:V01:METAR:obs:ingest:netcdf";
+USE KEYS ["MD:V01:METAR:obs:ingest:netcdf"];

tests/vxingest/netcdf_to_cb/testdata/test_ingest_document_id.n1ql

@@ -1,3 +1,3 @@
 Select ingest_document_ids
 from `vxdata`._default.METAR
-where meta().id = "MD:V01:METAR:obs:ingest:netcdf";
+USE KEYS ["MD:V01:METAR:obs:ingest:netcdf"];

tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py

@@ -58,9 +58,9 @@ def test_check_fcst_valid_epoch_fcst_valid_iso():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     options = ClusterOptions(PasswordAuthenticator(_user, _password))
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     stmnt = f"""SELECT m0.fcstValidEpoch fve, fcstValidISO fvi
         FROM `{_bucket}`.{_scope}.{_collection} m0
         WHERE
@@ -105,7 +105,7 @@ def test_get_stations_geo_search():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     collection = cluster.bucket(_bucket).scope(_scope).collection(_collection)
     load_spec = {}
     load_spec["cluster"] = cluster
@@ -273,7 +273,7 @@ def test_ps_surface_data_hrrr_ops_all_hrrr():
     options = ClusterOptions(
         PasswordAuthenticator(_user, _password), timeout_options=timeout_options
     )
-    cluster = Cluster("couchbase://" + _host, options)
+    cluster = Cluster(_host, options)
     # get available fcstValidEpochs for couchbase
 
     result = cluster.query(

tests/vxingest/partial_sums_to_cb/test_unit_queries.py

@@ -35,7 +35,7 @@ def connect_cb():
         PasswordAuthenticator(cb_connection["user"], cb_connection["password"]),
         timeout_options=timeout_options,
     )
-    cb_connection["cluster"] = Cluster("couchbase://" + cb_connection["host"], options)
+    cb_connection["cluster"] = Cluster(cb_connection["host"], options)
     cb_connection["collection"] = (
         cb_connection["cluster"]
         .bucket(cb_connection["bucket"])
@@ -145,7 +145,7 @@ def test_get_stations(request):
 @pytest.mark.integration()
 def test_get_threshold_descriptions(request):
     _name = request.node.name
-    _expected_time = 0.01
+    _expected_time = 0.6
     testdata = Path(
         "tests/vxingest/partial_sums_to_cb/testdata/test_get_threshold_descriptions.n1ql"
     )