fix linting errors and go ahead and add two utilitiy scripts

src/vxingest/ctc_to_cb/ctc_builder.py

@@ -17,7 +17,6 @@
 
 from couchbase.exceptions import DocumentNotFoundException, TimeoutException
 from couchbase.search import GeoBoundingBoxQuery, SearchOptions
-
 from vxingest.builder_common.builder import Builder
 from vxingest.builder_common.builder_utilities import (
     convert_to_iso,

src/vxingest/grib2_to_cb/grib_builder.py

@@ -13,7 +13,6 @@
 from pathlib import Path
 
 import numpy as np
-
 from vxingest.builder_common.builder_utilities import get_geo_index
 from vxingest.grib2_to_cb.grib_builder_parent import GribBuilder
 

src/vxingest/main.py

@@ -23,7 +23,6 @@
     QueryOptions,
 )
 from prometheus_client import CollectorRegistry, Counter, Gauge, write_to_textfile
-
 from vxingest.ctc_to_cb.run_ingest_threads import VXIngest as CTCIngest
 from vxingest.grib2_to_cb.run_ingest_threads import VXIngest as GRIBIngest
 from vxingest.log_config import (

src/vxingest/netcdf_to_cb/netcdf_builder.py

@@ -22,7 +22,6 @@
 import numpy.ma as ma
 from metpy.calc import relative_humidity_from_dewpoint, wind_components
 from metpy.units import units
-
 from vxingest.builder_common.builder import Builder
 from vxingest.builder_common.builder_utilities import (
     convert_to_iso,

src/vxingest/partial_sums_to_cb/partial_sums_builder.py

@@ -19,7 +19,6 @@
 from couchbase.search import GeoBoundingBoxQuery, SearchOptions
 from metpy.calc import relative_humidity_from_dewpoint, wind_components
 from metpy.units import units
-
 from vxingest.builder_common.builder import Builder
 from vxingest.builder_common.builder_utilities import (
     convert_to_iso,

tests/vxingest/builder_common/test_bc_builder_utilities.py

@@ -1,5 +1,4 @@
 import pytest
-
 from vxingest.builder_common.builder_utilities import (
     convert_to_iso,
     get_geo_index,

tests/vxingest/builder_common/test_unit_queries.py

@@ -45,7 +45,7 @@ def connect_cb():
     return cb_connection
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_get_file_list(request):
     vx_ingest = VXIngest()
     vx_ingest.credentials_file = os.environ["CREDENTIALS"]
@@ -57,13 +57,13 @@ def test_get_file_list(request):
     temp_dir_path = "/tmp/vx_ingest"
     shutil.rmtree(temp_dir_path, ignore_errors=True)
     Path(temp_dir_path).mkdir(parents=True, exist_ok=True)
-    with Path(os.path.join(temp_dir_path, "2128723000010")).open("w") as f:
+    with Path(temp_dir_path / "2128723000010").open("w") as f:
         f.write("test")
-    with open(os.path.join(temp_dir_path, "2128723000020"), "w") as f:
+    with Path.open(Path(temp_dir_path / "2128723000020"), "w") as f:
         f.write("test")
-    with open(os.path.join(temp_dir_path, "2128723000030"), "w") as f:
+    with Path.open(Path(temp_dir_path / "2128723000030"), "w") as f:
         f.write("test")
-    with open(os.path.join(temp_dir_path, "2128723000040"), "w") as f:
+    with Path.open(Path(temp_dir_path, "2128723000040"), "w") as f:
         f.write("test")
 
     file_list = vx_ingest.get_file_list(
@@ -76,7 +76,7 @@ def test_get_file_list(request):
     ), f"{request.node.name}: file_list is not reverse sorted"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_stations_fcst_valid_epoch(request):
     _expected_time = 10
     _name = request.node.name
@@ -96,7 +96,7 @@ def test_stations_fcst_valid_epoch(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_stations_get_file_list_grib2(request):
     _expected_time = 16
     _name = request.node.name
@@ -114,7 +114,7 @@ def test_stations_get_file_list_grib2(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_stations_get_file_list_netcdf(request):
     _expected_time = 5
     _name = request.node.name
@@ -132,7 +132,7 @@ def test_stations_get_file_list_netcdf(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_metar_count(request):
     _expected_time = 0.05
     _name = request.node.name

tests/vxingest/ctc_to_cb/test_int_metar_ctc.py

@@ -14,7 +14,6 @@
 from couchbase.auth import PasswordAuthenticator
 from couchbase.cluster import Cluster
 from couchbase.options import ClusterOptions, ClusterTimeoutOptions
-
 from vxingest.ctc_to_cb import ctc_builder
 from vxingest.ctc_to_cb.run_ingest_threads import VXIngest
 
@@ -38,7 +37,7 @@ def stub_worker_log_configurer(queue: Queue):
     pass
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_check_fcst_valid_epoch_fcst_valid_iso():
     """
     integration test to check fcst_valid_epoch is derived correctly
@@ -84,7 +83,7 @@ def test_check_fcst_valid_epoch_fcst_valid_iso():
         assert (fve % 3600) == 0, "fcstValidEpoch is not at top of hour"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_get_stations_geo_search():
     """
     Currently we know that there are differences between the geo search stations list and the legacy
@@ -289,7 +288,7 @@ def calculate_cb_ctc(
     return ctc
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_ctc_builder_ceiling_hrrr_ops_all_hrrr():
     """
     This test verifies that data is returned for each fcstLen and each threshold.
@@ -381,7 +380,7 @@ def test_ctc_builder_ceiling_hrrr_ops_all_hrrr():
                 continue
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_ctc_builder_visibility_hrrr_ops_all_hrrr():
     """
     This test verifies that data is returned for each fcstLen and each threshold.
@@ -473,7 +472,7 @@ def test_ctc_builder_visibility_hrrr_ops_all_hrrr():
                 continue
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_ctc_ceiling_data_hrrr_ops_all_hrrr():
     """
     This test is a comprehensive test of the ctcBuilder data. It will retrieve CTC documents
@@ -606,7 +605,7 @@ def test_ctc_ceiling_data_hrrr_ops_all_hrrr():
                 the derived CTC {field}: {_ctc_value} and caclulated CTC {field}: {_cb_ctc_value} values do not match"""
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_ctc_visibiltiy_data_hrrr_ops_all_hrrr():
     """
     This test is a comprehensive test of the ctcBuilder data. It will retrieve CTC documents

tests/vxingest/ctc_to_cb/test_unit_metar_ctc.py

@@ -2,7 +2,6 @@
 from multiprocessing import JoinableQueue
 
 import pytest
-
 from vxingest.ctc_to_cb.run_ingest_threads import VXIngest
 from vxingest.ctc_to_cb.vx_ingest_manager import VxIngestManager
 
@@ -32,7 +31,7 @@ def setup_ingest():
     return _vx_ingest, vx_ingest_manager
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_cb_connect_disconnect():
     """test the cb connect and close"""
     vx_ingest_manager = None
@@ -51,7 +50,7 @@ def test_cb_connect_disconnect():
             vx_ingest_manager.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_credentials_and_load_spec():
     """test the get_credentials and load_spec"""
     vx_ingest_manager = None
@@ -66,7 +65,7 @@ def test_credentials_and_load_spec():
             vx_ingest_manager.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_write_load_job_to_files(tmp_path):
     """test write the load job"""
     vx_ingest_manager = None
@@ -83,7 +82,7 @@ def test_write_load_job_to_files(tmp_path):
             vx_ingest_manager.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_build_load_job_doc():
     """test the build load job"""
     vx_ingest_manager = None

tests/vxingest/ctc_to_cb/test_unit_queries_ctc.py

@@ -44,7 +44,7 @@ def connect_cb():
     return cb_connection
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_epoch_fcstlen_model(request):
     _name = request.node.name
     _expected_time = 3.0
@@ -62,7 +62,7 @@ def test_epoch_fcstlen_model(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_epoch_fcstlen_obs(request):
     _name = request.node.name
     _expected_time = 0.2
@@ -80,7 +80,7 @@ def test_epoch_fcstlen_obs(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_forecast_valid_epoch(request):
     _name = request.node.name
     _expected_time = 6.0
@@ -98,7 +98,7 @@ def test_forecast_valid_epoch(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_get_region_lat_lon(request):
     _name = request.node.name
     _expected_time = 0.01
@@ -116,7 +116,7 @@ def test_get_region_lat_lon(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_get_stations(request):
     _name = request.node.name
     _expected_time = 3
@@ -134,7 +134,7 @@ def test_get_stations(request):
     ), f"{_name}: elasped_time greater than {_expected_time} {elapsed_time}"
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_get_threshold_descriptions(request):
     _name = request.node.name
     _expected_time = 0.6

tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py

@@ -18,7 +18,6 @@
 from couchbase.auth import PasswordAuthenticator
 from couchbase.cluster import Cluster
 from couchbase.options import ClusterOptions, ClusterTimeoutOptions
-
 from vxingest.grib2_to_cb.run_ingest_threads import VXIngest
 
 cb_connection = {}
@@ -65,7 +64,7 @@ def connect_cb():
         return cb_connection
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_grib_builder_one_thread_file_pattern_hrrr_ops_conus(tmp_path):
     """test gribBuilder with one thread.
     This test verifies the resulting data file against the one that is in couchbase already
@@ -176,7 +175,7 @@ def test_grib_builder_one_thread_file_pattern_hrrr_ops_conus(tmp_path):
                     {_k}.{_dk} {result['data'][_k][_dk]} != {_json['data'][_k][_dk]} within {abs_tol} decimal places."""
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_grib_builder_two_threads_file_pattern_hrrr_ops_conus(tmp_path):
     """test gribBuilder multi-threaded
     Not going to qulify the data on this one, just make sure it runs two threads properly
@@ -208,7 +207,7 @@ def test_grib_builder_two_threads_file_pattern_hrrr_ops_conus(tmp_path):
     )
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_grib_builder_two_threads_file_pattern_rap_ops_130_conus(tmp_path):
     """test gribBuilder multi-threaded
     Not going to qulify the data on this one, just make sure it runs two threads properly

tests/vxingest/grib2_to_cb/test_unit_metar_model_grib.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 
 import pytest
-
 from vxingest.grib2_to_cb.run_ingest_threads import VXIngest
 
 
@@ -30,7 +29,7 @@ def setup_connection():
     return _vx_ingest
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_credentials_and_load_spec():
     """test the get_credentials and load_spec"""
     vx_ingest = None
@@ -43,7 +42,7 @@ def test_credentials_and_load_spec():
         vx_ingest.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_credentials_and_load_spec_multiple_ingest_ids():
     """test the get_credentials and load_spec"""
     vx_ingest = None
@@ -56,7 +55,7 @@ def test_credentials_and_load_spec_multiple_ingest_ids():
         vx_ingest.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_cb_connect_disconnect():
     """test the cb connect and close"""
     vx_ingest = None
@@ -72,7 +71,7 @@ def test_cb_connect_disconnect():
         vx_ingest.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_write_load_job_to_files(tmp_path):
     """test write the load job"""
     vx_ingest = None
@@ -88,7 +87,7 @@ def test_write_load_job_to_files(tmp_path):
         vx_ingest.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_build_load_job_doc(tmp_path):
     """test the build load job"""
     vx_ingest = None
@@ -108,7 +107,7 @@ def test_build_load_job_doc(tmp_path):
         vx_ingest.close_cb()
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_vxingest_get_file_list(tmp_path):
     """test the vxingest get_file_list"""
     vx_ingest = None

tests/vxingest/grib2_to_cb/test_unit_proj.py

@@ -3,7 +3,6 @@
 import pyproj
 import pytest
 import xarray as xr
-
 from vxingest.grib2_to_cb.run_ingest_threads import VXIngest
 
 
@@ -19,7 +18,7 @@ def setup_connection():
     return _vx_ingest
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_proj():
     """test the proj"""
     vx_ingest = None

tests/vxingest/grib2_to_cb/test_unit_proj_nocb.py

@@ -3,7 +3,7 @@
 import xarray as xr
 
 
-@pytest.mark.integration
+@pytest.mark.integration()
 def test_proj_nocb():
     """test the proj"""
     ds_height_above_ground_2m = xr.open_dataset(