261-erroneous-values-in-database-for-moisture-variables (#285)

This PR should add changes to the netcdf builder to accomodate RH,
WindU, and WindV values and also a backfill scripot, along with some
other minor changes.

mats_metadata_and_indexes/index_creation_scripts/README.md

@@ -1,11 +1,21 @@
-# How to build the indexes
+# Indexes
+
+## Overview
+
+### Cluster roles
+
+#### ingest / development
+The standalone cluster on adb-cb1 is used for ingest data creation and import. The ingest processes query this cluster and the import processes import to this cluster.
+#### production
+The production cluster is what the apps read. This cluster is populated by xdcr from the ingest cluster
 
 ## Index Creation Scripts
+We are trying to keep an up to date N1QL script for the standalone cluster and a different one for the production cluster (they are not the same index set) that contain the index creation statements necessary to recreate a proper set of indexes for either of those environments, or for a new environment that has one of thse roles. 
 
 There are three index creation scripts
 
 - mats_metadata_and_indexes/index_creation_scripts/create_indexes-0-replicas.n1ql
-  This is for the standalone server since it cannot support index replication (only one node)
+  This is for the standalone server since it cannot support index replication (it only has one node)
   Copy the contents of this N1QL script and either execute it in the query window of the UI of the
   standalone server or execute it with the command line interface cbq.
   After that you must also execute the N1QL in the build_indexes.n1ql script.
@@ -18,3 +28,6 @@ There are three index creation scripts
     This is for creating the full text search index. You can execute it as a bash script from any machine
     that has connectivity to the couchbase server. This script takes one parameter - the full hostname of 
     the couchbase server. It will prompt for the avid user password.
+
+## Index Backups
+It is difficult to keep these scripts up to date because it is essentially a manual operation. For that reason we keep a set of backups. These backups are small files that are derived from the curl based admin interface for Couchbase.

mats_metadata_and_indexes/index_creation_scripts/get_index_definitions.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+if [ $# -ne 1 ]; then
+  echo "Usage $0 credentials-file"
+  exit 1
+fi
+if [[ ! -f "$1" ]]; then
+  echo "$1 is not a file - exiting"
+  exit 1
+fi
+credentials_file=$1
+
+host=`grep cb_host ${credentials_file} | awk '{print $2}'`
+user=`grep cb_user ${credentials_file} | awk '{print $2}'`
+pwd=`grep cb_password ${credentials_file} | awk '{print $2}'`
+
+cred="${user}:${pwd}"
+
+curl --user ${cred} "http://${host}:8091/indexStatus" | jq -r '.indexes | .[] .definition'

scripts/admin/cluster/rbac_export_import.py

@@ -12,9 +12,11 @@
 # example restore: python rbac.py --restore
 
 import argparse
-import requests
 import json
 
+import requests
+
+
 def backup():
     url = 'http://' + args.host + '/settings/rbac/users'
     print("my url : " + str(url))
@@ -35,7 +37,7 @@ def backup():
             for entry in records:
                 entry['password'] = "password"
                 rbacInfo['userInfo'].append(entry)
-            with open(args.rbacFile, 'wt') as f:
+            with open(args.rbacFile, "w") as f:
                 f.write(json.dumps(rbacInfo, indent=4, sort_keys=True))
                 print('Successfully backed up RBAC data to: [%s]' % args.rbacFile)
         except Exception as e:
@@ -45,7 +47,7 @@ def backup():
 
 def restore():
     try:
-        with open(args.rbacFile,'r') as f:
+        with open(args.rbacFile) as f:
             rbacData = json.loads(f.read())
             for cluster in rbacData['clusterInfo']['cluster']:
                 print(cluster)
@@ -69,7 +71,7 @@ def restore():
                         print(output + json.dumps(params))
                     else:
                         print(params)
-                        print('Http requests {} failed: {} {}'.format(url, response.status_code, response.text))
+                        print(f'Http requests {url} failed: {response.status_code} {response.text}')
 
     except Exception as e:
                 print('Error:' + str(e.args))

scripts/admin/index/index-analysis.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-# -*- coding: utf-8 -*-
 
 # pylint: disable=C0303, C0325
 

scripts/admin/stats/StoreAndGetStats/getStats/bucket.py

@@ -1,5 +1,4 @@
-import json
-class Bucket(object):
+class Bucket:
     def __init__(self, attributes_list):
         self.attribute_map  = {}
         self.attributes_list = attributes_list

scripts/admin/stats/StoreAndGetStats/getStats/cbcluster.py

@@ -1,8 +1,8 @@
-from couchbase.cluster import Cluster
-from couchbase.cluster import PasswordAuthenticator
+from couchbase.cluster import Cluster, PasswordAuthenticator
 from couchbase.n1ql import N1QLQuery
 
-class CBCluster(object):
+
+class CBCluster:
 	def __init__(self, attributes_list, ipaddr):
 		cluster = Cluster('couchbase://' + ipaddr)
 		authenticator = PasswordAuthenticator(attributes_list[unicode("user")], attributes_list[unicode("pwd")])

scripts/admin/stats/StoreAndGetStats/getStats/fileOutput.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 from nodeOutput import NodeOutput
 
-class FileOutput(object):
+
+class FileOutput:
 	header_line = "\"Date Time\""
 	quota_used = "\"quotaPercentUsed\""
 	itemCount = "\"itemCount\""

scripts/admin/stats/StoreAndGetStats/getStats/node.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-class Node(object):
+class Node:
 
     def __init__(self, attributes_dict, timeslice):
         self.attributes_dict = attributes_dict

scripts/admin/stats/StoreAndGetStats/getStats/nodeOutput.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-class NodeOutput(object):
+class NodeOutput:
 
 	def __init__(self, nodeName):
 		self.nodeName = nodeName

scripts/admin/stats/StoreAndGetStats/getStats/timeSlice.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-class TimeSlice(object):
+class TimeSlice:
 
 	def __init__(self, id, timeslice):
 		self.id = id

scripts/admin/stats/StoreAndGetStats/storeStats/bucket.py

@@ -1,5 +1,4 @@
-import json
-class Bucket(object):
+class Bucket:
     def __init__(self, attributes_list):
         self.attribute_map  = {}
         self.attributes_list = attributes_list

scripts/admin/stats/StoreAndGetStats/storeStats/cbcluster.py

@@ -1,6 +1,7 @@
-from couchbase.cluster import Cluster
-from couchbase.cluster import PasswordAuthenticator
-class CBCluster(object):
+from couchbase.cluster import Cluster, PasswordAuthenticator
+
+
+class CBCluster:
 	def __init__(self, attributes_list, ipaddr):
 		cluster = Cluster('couchbase://' + ipaddr)
 		authenticator = PasswordAuthenticator(attributes_list[unicode("user")], attributes_list[unicode("pwd")])

scripts/backup/automate_backup.py

@@ -8,15 +8,17 @@
 __maintainer__ = "tdenton"
 __version__ = "0.0.1"
 
-import subprocess
-from datetime import datetime
-import shutil
 import argparse
-import os
 import json
+import os
+import shutil
+import subprocess
+from datetime import datetime
+
 from config import Config
 
-class BackerUpper(object):
+
+class BackerUpper:
     '''This class is what actually does the backing up, archiving and cleaning of old backups'''
     def __init__(self, hostname="", port="", username="", password=""):
         self.instance = self.get_config()
@@ -60,7 +62,7 @@ def get_repo_name(self):
 
     def archive_repo(self):
         '''Copys the existing current backup and turns it into an archive'''
-        shutil.copytree("{}/{}".format(self.instance.archive, self.instance.repo), 
+        shutil.copytree(f"{self.instance.archive}/{self.instance.repo}", 
                         "{}/{}".format(self.instance.archive, datetime.now().strftime("%Y-%m-%d")),
                         symlinks=False,
                         ignore=None)
@@ -77,15 +79,15 @@ def backup_repo(self):
                         "-r", 
                         self.instance.repo, 
                         "-c", 
-                        "{}:{}".format(self.instance.hostname, self.instance.port),
+                        f"{self.instance.hostname}:{self.instance.port}",
                         "-u",
                         self.instance.username,
                         "-p",
                         self.instance.password]
         if self.instance.value_compression != "unchanged":
-            backup_array.append("--value_compression {}".format(self.instance.value_compression))
+            backup_array.append(f"--value_compression {self.instance.value_compression}")
         if self.instance.threads != 1:
-            backup_array.append("--threads {}".format(self.instance.threads))
+            backup_array.append(f"--threads {self.instance.threads}")
         if self.instance.no_progress_bar:
             backup_array.append("--no-progress-bar")
         try:

scripts/backup/config.py

@@ -1,6 +1,6 @@
 # pyline disable=C0303, C0325
 
-class Config(object):
+class Config:
     def __init__(self, my_json):
         self.my_json = my_json
         self.weeks_to_keep = self.my_json['weeks_to_keep']