DB Backup Script Fix (#3131)

* - Updated script to correctly access the DB name and create the backup

* - fix lint
- update logic

* - skipping pipeline
- using db name in backup name

* - Added new env var for deployments
- UPdated scrpt to support new names

* - revert

---------

Co-authored-by: Alex P. <[email protected]>

scripts/deploy-backend.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ##############################
-# Global Variable Decls 
+# Global Variable Decls
 ##############################
 
 # The deployment strategy you wish to employ ( rolling update or setting up a new environment)
@@ -77,15 +77,15 @@ set_cf_envs()
     else
       cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
     fi
-    
+
     echo "Setting var : $var_name"
     $cf_cmd
   done
 
 }
 
 # Helper method to generate JWT cert and keys for new environment
-generate_jwt_cert() 
+generate_jwt_cert()
 {
     echo "regenerating JWT cert/key"
     yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256
@@ -94,7 +94,7 @@ generate_jwt_cert()
 }
 
 update_kibana()
-{  
+{
   # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
   cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
   cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
@@ -105,12 +105,16 @@ update_backend()
 {
     cd tdrs-backend || exit
     cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
-    
+
     if [ "$CF_SPACE" = "tanf-prod" ]; then
       cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
     else
       # Add environment varilables for clamav
       cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan"
+
+      # Add variable for dev/staging apps to know their DB name. Prod uses default AWS name.
+      cf unset-env "$CGAPPNAME_BACKEND" "APP_DB_NAME"
+      cf set-env "$CGAPPNAME_BACKEND" "APP_DB_NAME" "tdp_db_$backend_app_name"
     fi
 
     if [ "$1" = "rolling" ] ; then
@@ -129,12 +133,12 @@ update_backend()
     fi
 
     set_cf_envs
-    
+
     cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND"
 
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
-    
+
     if [ "$CF_SPACE" = "tanf-prod" ]; then
       # Add network policy to allow backend to access tanf-prod services
       cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
@@ -149,7 +153,7 @@ bind_backend_to_services() {
     echo "Binding services to app: $CGAPPNAME_BACKEND"
 
     if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then
-      # TODO: this is technical debt, we should either make staging mimic tanf-dev 
+      # TODO: this is technical debt, we should either make staging mimic tanf-dev
       #       or make unique services for all apps but we have a services limit
       #       Introducing technical debt for release 3.0.0 specifically.
       env="develop"
@@ -158,10 +162,10 @@ bind_backend_to_services() {
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${env}"
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}"
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}"
-    
+
     # Setting up the ElasticSearch service
     cf bind-service "$CGAPPNAME_BACKEND" "es-${env}"
-    
+
     set_cf_envs
 
     echo "Restarting app: $CGAPPNAME_BACKEND"

tdrs-backend/tdpservice/scheduling/management/db_backup.py

@@ -57,28 +57,16 @@ def get_system_values():
     sys_values['S3_SECRET_ACCESS_KEY'] = sys_values['S3_CREDENTIALS']['secret_access_key']
     sys_values['S3_BUCKET'] = sys_values['S3_CREDENTIALS']['bucket']
     sys_values['S3_REGION'] = sys_values['S3_CREDENTIALS']['region']
-    sys_values['DATABASE_URI'] = OS_ENV['DATABASE_URL']
+
     # Set AWS credentials in env, Boto3 uses the env variables for connection
     os.environ["AWS_ACCESS_KEY_ID"] = sys_values['S3_ACCESS_KEY_ID']
     os.environ["AWS_SECRET_ACCESS_KEY"] = sys_values['S3_SECRET_ACCESS_KEY']
 
     # Set Database connection info
-    AWS_RDS_SERVICE_JSON = json.loads(OS_ENV['VCAP_SERVICES'])['aws-rds'][0]['credentials']
-    sys_values['DATABASE_PORT'] = AWS_RDS_SERVICE_JSON['port']
-    sys_values['DATABASE_PASSWORD'] = AWS_RDS_SERVICE_JSON['password']
-    sys_values['DATABASE_DB_NAME'] = AWS_RDS_SERVICE_JSON['db_name']
-    sys_values['DATABASE_HOST'] = AWS_RDS_SERVICE_JSON['host']
-    sys_values['DATABASE_USERNAME'] = AWS_RDS_SERVICE_JSON['username']
-
-    # write .pgpass
-    with open('/home/vcap/.pgpass', 'w') as f:
-        f.write(sys_values['DATABASE_HOST'] + ":"
-                + sys_values['DATABASE_PORT'] + ":"
-                + settings.DATABASES['default']['NAME'] + ":"
-                + sys_values['DATABASE_USERNAME'] + ":"
-                + sys_values['DATABASE_PASSWORD'])
-    os.environ['PGPASSFILE'] = '/home/vcap/.pgpass'
-    os.system('chmod 0600 /home/vcap/.pgpass')
+    AWS_RDS_SERVICE_JSON = json.loads(OS_ENV['VCAP_SERVICES'])['aws-rds'][0]
+    sys_values['DATABASE_URI'] = AWS_RDS_SERVICE_JSON['credentials']['uri'].rsplit('/', 1)[0]
+    sys_values['DATABASE_DB_NAME'] = AWS_RDS_SERVICE_JSON['credentials']['db_name']
+
     return sys_values
 
 
@@ -94,19 +82,11 @@ def backup_database(file_name,
     pg_dump -F c --no-acl --no-owner -f backup.pg postgresql://${USERNAME}:${PASSWORD}@${HOST}:${PORT}/${NAME}
     """
     try:
-        # TODO: This is a bandaid until the correct logic is determined for the system values with respect to the
-        # correct database name.
-        # cmd = postgres_client + "pg_dump -Fc --no-acl -f " + file_name + " -d " + database_uri
-        db_host = settings.DATABASES['default']['HOST']
-        db_port = settings.DATABASES['default']['PORT']
-        db_name = settings.DATABASES['default']['NAME']
-        db_user = settings.DATABASES['default']['USER']
-
-        export_password = f"export PGPASSWORD={settings.DATABASES['default']['PASSWORD']}"
-        cmd = (f"{export_password} && {postgres_client}pg_dump -h {db_host} -p {db_port} -d {db_name} -U {db_user} "
-               f"-F c --no-password --no-acl --no-owner -f {file_name}")
+        cmd = f"{postgres_client}pg_dump -Fc --no-acl -f {file_name} -d {database_uri}"
         logger.info(f"Executing backup command: {cmd}")
-        os.system(cmd)
+        code = os.system(cmd)
+        if code != 0:
+            raise Exception("pg_dump command failed with a non zero exit code.")
         msg = "Successfully executed backup. Wrote pg dumpfile to {}".format(file_name)
         logger.info(msg)
         LogEntry.objects.log_action(
@@ -268,28 +248,47 @@ def get_database_credentials(database_uri):
     database_name = database_uri
     return [username, password, host, port, database_name]
 
-
-def main(argv, sys_values, system_user):
-    """Handle commandline args."""
-    arg_file = "/tmp/backup.pg"
-    arg_database = sys_values['DATABASE_URI']
+def get_opts(argv, db_name):
+    """Parse command line options."""
+    arg_file = f"/tmp/{db_name}_backup.pg"
     arg_to_restore = False
     arg_to_backup = False
+    restore_db_name = None
+    opts, args = getopt.getopt(argv, "hbrf:n:", ["help", "backup", "restore", "file=", "restore_db_name="])
+    for opt, arg in opts:
+        if "backup" in opt or "-b" in opt:
+            arg_to_backup = True
+        elif "restore" in opt or "-r" in opt:
+            arg_to_restore = True
+        if "file" in opt or "-f" in opt and arg:
+            arg_file = arg if arg[0] == "/" else "/tmp/" + arg
+        if "restore_db_name" in opt or "-n" in opt and arg:
+            restore_db_name = arg
+
+    if arg_to_restore and not restore_db_name:
+        raise ValueError("You must pass a `-n <DB_NAME>` when trying to restore a DB.")
+
+    return arg_file, arg_to_backup, arg_to_restore, restore_db_name
+
+def get_db_name(sys_values):
+    """
+    Get the correct database name.
 
-    try:
-        opts, args = getopt.getopt(argv, "hbrf:d:", ["help", "backup", "restore", "file=", "database=", ])
-        for opt, arg in opts:
-            if "backup" in opt or "-b" in opt:
-                arg_to_backup = True
-            elif "restore" in opt or "-r" in opt:
-                arg_to_restore = True
-            if "file" in opt or "-f" in opt and arg:
-                arg_file = arg if arg[0] == "/" else "/tmp/" + arg
-            if "database" in opt or "-d" in opt:
-                arg_database = arg
+    In prod we use the default database name that AWS creates. In the Dev and Staging environments the databases are
+    named based off of their app; i.e. tdp_db_raft. The deploy script sets the APP_DB_NAME environment variable for all
+    apps except prod.
+    """
+    env_db_name = os.getenv("APP_DB_NAME", None)
+    if env_db_name is None:
+        return sys_values['DATABASE_DB_NAME']
+    return env_db_name
 
-    except Exception as e:
-        raise e
+def main(argv, sys_values, system_user):
+    """Handle commandline args."""
+    db_base_uri = sys_values['DATABASE_URI']
+
+    db_name = get_db_name(sys_values)
+    arg_file, arg_to_backup, arg_to_restore, restore_db_name = get_opts(argv, db_name)
 
     if arg_to_backup:
         LogEntry.objects.log_action(
@@ -303,7 +302,7 @@ def main(argv, sys_values, system_user):
         # back up database
         backup_database(file_name=arg_file,
                         postgres_client=sys_values['POSTGRES_CLIENT_DIR'],
-                        database_uri=arg_database,
+                        database_uri=f"{db_base_uri}/{db_name}",
                         system_user=system_user)
 
         # upload backup file
@@ -348,7 +347,7 @@ def main(argv, sys_values, system_user):
         # restore database
         restore_database(file_name=arg_file,
                          postgres_client=sys_values['POSTGRES_CLIENT_DIR'],
-                         database_uri=arg_database,
+                         database_uri=f"{db_base_uri}/{restore_db_name}",
                          system_user=system_user)
 
         LogEntry.objects.log_action(