261-erroneous-values-in-database-for-moisture-variables

pyproject.toml

@@ -24,6 +24,7 @@ prometheus-client = "^0.19.0"
 pytest = "^7.4.3"
 types-pyyaml = "^6.0.12.12"
 ruff = "^0.1.6"
+ipykernel = "^6.27.1"
 
 [build-system]
 requires = ["poetry-core"]

scripts/admin/cluster/rbac_export_import.py

@@ -12,9 +12,11 @@
 # example restore: python rbac.py --restore
 
 import argparse
-import requests
 import json
 
+import requests
+
+
 def backup():
     url = 'http://' + args.host + '/settings/rbac/users'
     print("my url : " + str(url))
@@ -35,7 +37,7 @@ def backup():
             for entry in records:
                 entry['password'] = "password"
                 rbacInfo['userInfo'].append(entry)
-            with open(args.rbacFile, 'wt') as f:
+            with open(args.rbacFile, "w") as f:
                 f.write(json.dumps(rbacInfo, indent=4, sort_keys=True))
                 print('Successfully backed up RBAC data to: [%s]' % args.rbacFile)
         except Exception as e:
@@ -45,7 +47,7 @@ def backup():
 
 def restore():
     try:
-        with open(args.rbacFile,'r') as f:
+        with open(args.rbacFile) as f:
             rbacData = json.loads(f.read())
             for cluster in rbacData['clusterInfo']['cluster']:
                 print(cluster)
@@ -69,7 +71,7 @@ def restore():
                         print(output + json.dumps(params))
                     else:
                         print(params)
-                        print('Http requests {} failed: {} {}'.format(url, response.status_code, response.text))
+                        print(f'Http requests {url} failed: {response.status_code} {response.text}')
 
     except Exception as e:
                 print('Error:' + str(e.args))

scripts/admin/index/index-analysis.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-# -*- coding: utf-8 -*-
 
 # pylint: disable=C0303, C0325
 

scripts/admin/stats/StoreAndGetStats/getStats/bucket.py

@@ -1,5 +1,4 @@
-import json
-class Bucket(object):
+class Bucket:
     def __init__(self, attributes_list):
         self.attribute_map  = {}
         self.attributes_list = attributes_list

scripts/admin/stats/StoreAndGetStats/getStats/cbcluster.py

@@ -1,8 +1,8 @@
-from couchbase.cluster import Cluster
-from couchbase.cluster import PasswordAuthenticator
+from couchbase.cluster import Cluster, PasswordAuthenticator
 from couchbase.n1ql import N1QLQuery
 
-class CBCluster(object):
+
+class CBCluster:
 	def __init__(self, attributes_list, ipaddr):
 		cluster = Cluster('couchbase://' + ipaddr)
 		authenticator = PasswordAuthenticator(attributes_list[unicode("user")], attributes_list[unicode("pwd")])

scripts/admin/stats/StoreAndGetStats/getStats/fileOutput.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 from nodeOutput import NodeOutput
 
-class FileOutput(object):
+
+class FileOutput:
 	header_line = "\"Date Time\""
 	quota_used = "\"quotaPercentUsed\""
 	itemCount = "\"itemCount\""

scripts/admin/stats/StoreAndGetStats/getStats/node.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-class Node(object):
+class Node:
 
     def __init__(self, attributes_dict, timeslice):
         self.attributes_dict = attributes_dict

scripts/admin/stats/StoreAndGetStats/getStats/nodeOutput.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-class NodeOutput(object):
+class NodeOutput:
 
 	def __init__(self, nodeName):
 		self.nodeName = nodeName

scripts/admin/stats/StoreAndGetStats/getStats/timeSlice.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-class TimeSlice(object):
+class TimeSlice:
 
 	def __init__(self, id, timeslice):
 		self.id = id

scripts/admin/stats/StoreAndGetStats/storeStats/bucket.py

@@ -1,5 +1,4 @@
-import json
-class Bucket(object):
+class Bucket:
     def __init__(self, attributes_list):
         self.attribute_map  = {}
         self.attributes_list = attributes_list

scripts/admin/stats/StoreAndGetStats/storeStats/cbcluster.py

@@ -1,6 +1,7 @@
-from couchbase.cluster import Cluster
-from couchbase.cluster import PasswordAuthenticator
-class CBCluster(object):
+from couchbase.cluster import Cluster, PasswordAuthenticator
+
+
+class CBCluster:
 	def __init__(self, attributes_list, ipaddr):
 		cluster = Cluster('couchbase://' + ipaddr)
 		authenticator = PasswordAuthenticator(attributes_list[unicode("user")], attributes_list[unicode("pwd")])

scripts/backup/automate_backup.py

@@ -8,15 +8,17 @@
 __maintainer__ = "tdenton"
 __version__ = "0.0.1"
 
-import subprocess
-from datetime import datetime
-import shutil
 import argparse
-import os
 import json
+import os
+import shutil
+import subprocess
+from datetime import datetime
+
 from config import Config
 
-class BackerUpper(object):
+
+class BackerUpper:
     '''This class is what actually does the backing up, archiving and cleaning of old backups'''
     def __init__(self, hostname="", port="", username="", password=""):
         self.instance = self.get_config()
@@ -60,7 +62,7 @@ def get_repo_name(self):
 
     def archive_repo(self):
         '''Copys the existing current backup and turns it into an archive'''
-        shutil.copytree("{}/{}".format(self.instance.archive, self.instance.repo), 
+        shutil.copytree(f"{self.instance.archive}/{self.instance.repo}", 
                         "{}/{}".format(self.instance.archive, datetime.now().strftime("%Y-%m-%d")),
                         symlinks=False,
                         ignore=None)
@@ -77,15 +79,15 @@ def backup_repo(self):
                         "-r", 
                         self.instance.repo, 
                         "-c", 
-                        "{}:{}".format(self.instance.hostname, self.instance.port),
+                        f"{self.instance.hostname}:{self.instance.port}",
                         "-u",
                         self.instance.username,
                         "-p",
                         self.instance.password]
         if self.instance.value_compression != "unchanged":
-            backup_array.append("--value_compression {}".format(self.instance.value_compression))
+            backup_array.append(f"--value_compression {self.instance.value_compression}")
         if self.instance.threads != 1:
-            backup_array.append("--threads {}".format(self.instance.threads))
+            backup_array.append(f"--threads {self.instance.threads}")
         if self.instance.no_progress_bar:
             backup_array.append("--no-progress-bar")
         try:

scripts/backup/config.py

@@ -1,6 +1,6 @@
 # pyline disable=C0303, C0325
 
-class Config(object):
+class Config:
     def __init__(self, my_json):
         self.my_json = my_json
         self.weeks_to_keep = self.my_json['weeks_to_keep']

scripts/troubleshooting/sdk/orphans_threshold.py

@@ -64,12 +64,14 @@
 If including a space between the date and time, enclose the
 datetime within quotes.
 """
-import json
-import csv
 import argparse
+import csv
+import json
 from datetime import datetime
+
 # pylint: disable=import-error
 import dictfier
+
 # pylint: enable=import-error
 
 # pylint: disable=superfluous-parens
@@ -82,7 +84,7 @@
 timeouts = []
 
 
-class Orphans():
+class Orphans:
     """
     This is the collection of orphan log entries
     that are extracted from the log file.
@@ -108,7 +110,7 @@ def __init__(self, log_line):
 
 
 # pylint: disable=too-many-instance-attributes
-class Orphan():
+class Orphan:
     """
     This is a single orphan log entry from
     the RTO log file.
@@ -125,7 +127,7 @@ def __init__(self, data_dict, log_time):
 # pylint: enable=too-many-instance-attributes
 
 
-class Thresholds():
+class Thresholds:
     """
     This is the collection of threshold log entries
     that are extracted from the log file.
@@ -151,7 +153,7 @@ def __init__(self, log_line, latency_threshold):
 
 
 # pylint: disable=too-many-instance-attributes
-class Threshold():
+class Threshold:
     """
     This is a single threshold log entry from
     the RTO log file.
@@ -195,7 +197,7 @@ def __init__(self, data_dict, log_time, latency_threshold):
 
 
 
-class Timeout():
+class Timeout:
     """
     This is the combined orphan and threshold log
     entries that will be written out to the .CSV file.
@@ -299,7 +301,7 @@ def read_log_file(log_file, start_date, end_date, threshold):
     threshold entries.
     """
     # Open and read in the file
-    input_file = open(log_file, "r")
+    input_file = open(log_file)
     # Iterate through the lines in the log file
     for line in input_file:
         # Extract the datetime from the line and compare it to

src/vxingest/netcdf_to_cb/netcdf_builder.py

@@ -20,7 +20,7 @@
 
 import netCDF4 as nc
 import numpy.ma as ma
-from metpy.calc import wind_components
+from metpy.calc import relative_humidity_from_dewpoint, wind_components
 from metpy.units import units
 from vxingest.builder_common.builder import Builder
 from vxingest.builder_common.builder_utilities import (
@@ -363,7 +363,7 @@ def build_document(self, queue_element):
             if self.do_profiling:
                 with cProfile.Profile() as _pr:
                     self.handle_document()
-                    with open("profiling_stats.txt", "w") as stream:
+                    with open("profiling_stats.txt", "w", encoding="utf-8" ) as stream:
                         stats = Stats(_pr, stream=stream)
                         stats.strip_dirs()
                         stats.sort_stats("time")
@@ -643,7 +643,26 @@ def umask_value_transform(self, params_dict):
             )
             return None
 
-    def handle_wind_dir_U(self, params_dict):
+    def handle_rh(self, params_dict):
+        """Retrieves a RH value, if there is one,
+           and if not derives the RH from the dewpoint and temperature.
+        Args:
+            params_dict (dict): named function parameters
+        Returns:
+            float: the RH
+        """
+        try:
+            # dewpoint = self.umask_value_transform({"recNum": params_dict["recNum"], "dewpoint":"dewpoint"})
+            # temperature = self.umask_value_transform({"recNum": params_dict["recNum"], "temperature":"temperature"})
+            dewpoint = self.umask_value_transform({"recNum": params_dict["recNum"], "dewpoint":params_dict["dewpoint"]})
+            temperature = self.umask_value_transform({"recNum": params_dict["recNum"], "temperature":params_dict["temperature"]})
+            _q = (relative_humidity_from_dewpoint(temperature * units.kelvin, dewpoint * units.kelvin).magnitude) * 100
+            return _q
+        except Exception as _e: # pylint:disable=broad-except
+            # there must not have been one
+            return None
+
+    def handle_wind_dir_u(self, params_dict):
         """Retrieves a wind direction U value, if there is one,
            and if not derives the U component from the wind direction and speed.
            expects wind speed and wind direction to be in the params_dict.
@@ -653,19 +672,24 @@ def handle_wind_dir_U(self, params_dict):
             float: the wind direction
         """
         try:
-            value = self.umask_value_transform(params_dict)
-            # value = self.meterspersecond_to_milesperhour({"recNum": rec_num, "windSpeed": ""})
-            if value is not None:
-                value = 360 - value
-            return value
-        except Exception as _e:
-            # there must not have been one
-            windDir = self.umask_value_transform({})
-            windSpeed = self.umask_value_transform({})
-            u, _v = wind_components(windSpeed * units("m/s"), windDir * units.deg)
+            _wind_dir = self.umask_value_transform({"recNum": params_dict["recNum"], "windDir":params_dict["windDir"]})
+            _wind_speed = self.umask_value_transform({"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]})
+            if _wind_dir is None:
+                return None
+            if _wind_speed is None:
+                return None
+            # wind speed is in meters per second and windDir is in degrees from netcdf file
+            u = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[0].magnitude
             return u
+        except Exception as _e: # pylint:disable=broad-exception-caught
+            logger.error(
+                "%s handle_wind_dir_v: Exception in named function:  error: %s",
+                self.__class__.__name__,
+                str(_e),
+            )
+            return None
 
-    def handle_wind_dir_V(self, params_dict):
+    def handle_wind_dir_v(self, params_dict):
         """Retrieves a wind direction V value, if there is one,
            and if not derives the V component from the wind direction and speed.
         Args:
@@ -674,16 +698,22 @@ def handle_wind_dir_V(self, params_dict):
             float: the wind direction
         """
         try:
-            value = self.umask_value_transform({})
-            if value is not None:
-                value = 360 - value
-            return value
-        except Exception as _e:
-            # there must not have been one
-            windDir = self.umask_value_transform({})
-            windSpeed = self.umask_value_transform({})
-            _u, v = wind_components(windSpeed * units("m/s"), windDir * units.deg)
+            _wind_dir = self.umask_value_transform({"recNum": params_dict["recNum"], "windDir":params_dict["windDir"]})
+            _wind_speed = self.umask_value_transform({"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]})
+            if _wind_dir is None:
+                return None
+            if _wind_speed is None:
+                return None
+            # wind speed is in meters per second and windDir is in degrees from netcdf file
+            v = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[1].magnitude
             return v
+        except Exception as _e: # pylint:disable=broad-exception-caught
+            logger.error(
+                "%s handle_wind_dir_v: Exception in named function:  error: %s",
+                self.__class__.__name__,
+                str(_e),
+            )
+            return None
 
     def handle_pressure(self, params_dict):
         """Retrieves a pressure value and converts it to millibars from pascals
@@ -908,7 +938,7 @@ def handle_station(self, params_dict):
                     "version": "V01",
                 }
                 # add the new station to the document map with the new id
-                if an_id not in self.document_map.keys():
+                if an_id not in self.document_map:
                     self.document_map[an_id] = new_station
                 self.stations.append(new_station)
             else:

src/vxingest/utilities/README.MD

@@ -0,0 +1,3 @@
+# VxIngest utilities
+
+These are scripts that have been used for various tasks, like backfilling data, that can be useful examples for other tasks.