Passing database name 1

README.md

@@ -13,4 +13,19 @@ The mtk config file should be provided as a base64 encoded string, this allows f
 ```
 $ cat example.mtk.yml | base64 | tr -d '\n\r'
 cmV3cml0ZToKICAjIERydXBhbCA4CiAgdXNlcnNfZmllbGRfZGF0YToKICAgIG1haWw6IGNvbmNhdCh1aWQsICJAU0FOSVRJU0VEIikKICAgIHBhc3M6ICciU0FOSVRJWkVEX1BBU1NXT1JEIicKICAgICMgV2UgZG9uJ3QgYWxsb3cgbm8gZGlydHkgbmFtZXMgYXJvdW5kIGhlcmUhCiAgICBuYW1lOiBjb25jYXQodWlkLCAiLVNBTklUSVNFRCIpCiAgICBpbml0OiAnIlNBTklUSVNFRF9JTklUIicKICAjIERydXBhbCA3CiAgdXNlcnM6CiAgICBtYWlsOiBjb25jYXQodWlkLCAiQFNBTklUSVNFRCIpCiAgICBwYXNzOiAnIlNBTklUSVpFRF9QQVNTV09SRCInCgp3aGVyZToKICAjIE9ubHkgaW5jbHVkZSBib2R5IGZpZWxkIGRhdGEgZm9yIGN1cnJlbnQgcmV2aXNpb25zLgogIG5vZGVfcmV2aXNpb25fX2JvZHk6IHwtCiAgICAgIHJldmlzaW9uX2lkIElOIChTRUxFQ1QgdmlkIEZST00gbm9kZSkKCm5vZGF0YToKICAtIGNhY2hlKgogIC0gY2FwdGNoYV9zZXNzaW9ucwogIC0gaGlzdG9yeQogIC0gZmxvb2QKICAtIGJhdGNoCiAgLSBxdWV1ZQogIC0gc2Vzc2lvbnMKICAtIHNlbWFwaG9yZQogIC0gc2VhcmNoX2FwaV90YXNrCiAgLSBzZWFyY2hfZGF0YXNldAogIC0gc2VhcmNoX2luZGV4CiAgLSBzZWFyY2hfdG90YWwKICAtIHdhdGNoZG9nCiAgLSB3ZWJmb3JtX3N1Ym1pc3Npb25fZGF0YQoKaWdub3JlOgogIC0gX19BQ1FVSUFfTU9OSVRPUklOR19fCgo=
-```
+```
+
+## Variables
+
+An example of the GraphQL needed to create an advanced task is available in 
+createDumpSanitisedDB.gql.  The variables which can be passed in are also 
+documented in this file (in the "displayName" field of each entry).  
+
+## Process
+
+The process goes through X basic stages:
+
+1. Set up all the initial variables
+2. MTK creates a database dump that's basically a sanitised .sql file
+3. Make docker-style container with sanitised DB (using podman)
+4. Save new container to registry

createDumpSanitisedDB.gql

@@ -62,7 +62,59 @@ mutation createDumpSanitisedDB {
 					displayName: "OPTIONAL: if you want to provide an organization base for the backup image name to parse",
 					type: STRING
 					optional: true
-				}
+				},
+				# Database hostname
+				{
+					name: "BUILDER_MTK_DUMP_HOSTNAME",
+					displayName: "OPTIONAL: The database hostname you want to use",
+					type: STRING
+					optional: true
+				},
+				{
+					name: "BUILDER_MTK_DUMP_HOSTNAME_NAME",
+					displayName: "OPTIONAL: The name of the variable that contains the database hostname you want to use",
+					type: STRING
+					optional: true
+				},
+				# Database name
+				{
+					name: "BUILDER_MTK_DUMP_DATABASE",
+					displayName: "OPTIONAL: The database name you want to use",
+					type: STRING
+					optional: true
+				},
+				{
+					name: "BUILDER_MTK_DUMP_DATABASE_NAME",
+					displayName: "OPTIONAL: The name of the variable that contains the database name you want to use",
+					type: STRING
+					optional: true
+				},
+				# Database username
+				{
+					name: "BUILDER_MTK_DUMP_USERNAME",
+					displayName: "OPTIONAL: The database username you want to use",
+					type: STRING
+					optional: true
+				},
+				{
+					name: "BUILDER_MTK_DUMP_USERNAME_NAME",
+					displayName: "OPTIONAL: The name of the variable that contains the database username you want to use",
+					type: STRING
+					optional: true
+				},
+				# Database password
+				{
+					name: "BUILDER_MTK_DUMP_PASSWORD",
+					displayName: "OPTIONAL: The database password you want to use",
+					type: STRING
+					optional: true
+				},
+				{
+					name: "BUILDER_MTK_DUMP_PASSWORD_NAME",
+					displayName: "OPTIONAL: The name of the variable that contains the database password you want to use",
+					type: STRING
+					optional: true
+				},
 			]
 		}
 	){
@@ -79,4 +131,4 @@ mutation createDumpSanitisedDB {
 			}
 		}
 	}
-}
+}

mariadb-image-builder

@@ -129,6 +129,98 @@ function projectEnvironmentVariableCheck() {
 	echo "$existingValue"
 }
 
+# function to return (echo) one of the MTK environment variables.  Order of things to check:
+# -	Existing value
+# -	Passed variable name
+# -	Created from LAGOON_SERVICE_NAME
+#
+# Parameters are:
+# -	<name>	Name of MTK variable to set
+# -	<search key>	Search key to use when searching JSON
+#
+# Also reads:
+# Variable                                           | Check
+# ---------------------------------------------------+--------------------------------
+# BUILDER_MTK_DUMP_<name>                            | projectEnvironmentVariableCheck
+# BUILDER_MTK_DUMP_<name>_NAME                       | projectEnvironmentVariableCheck
+# The variable named in BUILDER_MTK_DUMP_<name>_NAME | No
+# LAGOON_SERVICE_NAME                                | Should already be checked
+function calc_MTK_variable() {
+	local mtkVariableName="$1"
+	local jsonSearchKey="$2"
+
+	# check for argument
+	[ "$mtkVariableName" ] || return
+
+	##### If MTK_DUMP_* has been set
+	# Example values
+	# If MTK_DUMP_DATABASE = foodb
+	# $final_variable_name = MTK_DUMP_DATABASE
+	# ${!final_variable_name} = foodb
+	local final_variable_name="MTK_DUMP_${mtkVariableName}"
+	local source_variable_name="BUILDER_${final_variable_name}"
+	declare "$source_variable_name"=$(projectEnvironmentVariableCheck "$source_variable_name" "${!source_variable_name}" "$jsonSearchKey")
+
+	if [ -n "${!source_variable_name}" ]; then
+		echo "${!source_variable_name}"
+		return
+	fi
+
+	##### If MTK_DUMP_*_NAME has been set
+	# Example values
+	# If DB_NAME_CENTRAL=foodb and MTK_DUMP_DATABASE_NAME = DB_NAME_CENTRAL
+	# $nameholder_variable_name = MTK_DUMP_DATABASE_NAME
+	# $use_var = DB_NAME_CENTRAL
+	# ${!use_var} = foodb
+	local nameholder_variable_name="${final_variable_name}_NAME"
+	declare "$nameholder_variable_name"=$(projectEnvironmentVariableCheck "$nameholder_variable_name" "${!nameholder_variable_name}")
+
+	if [ -n "${!nameholder_variable_name}" ]; then
+		local use_var=${!nameholder_variable_name}
+		if [ "${!use_var}" ]; then
+			echo "${!use_var}"
+			return
+		else
+			echo "Warning: Could not find value for $nameholder_variable_name > $use_var" >&2
+		fi
+	fi
+
+	##### Special settings for special types
+	case $mtkVariableName in
+		DATABASE | USERNAME | PASSWORD)
+			# Do nothing
+			;;
+		HOSTNAME)
+			# Set the DB_X host variables to what would be the lagoon service variable values.  
+			# These are generated automatically from the LAGOON_SERVICE_NAME
+			# these will use bash references later; eg: ${!DB_HOST}
+			local DB_HOST="${LAGOON_SERVICE_NAME}_HOST"
+			local DB_READREPLICA_HOSTS="${LAGOON_SERVICE_NAME}_READREPLICA_HOSTS"
+
+			# Picks one of the read replicas to use when dumping the database
+			local DB_HOSTNAME
+			DB_HOSTNAME=$(echo "${!DB_READREPLICA_HOSTS}" | perl -F, -ane '$index=int(rand(scalar(@F)));print $F[$index]')
+			# if `DB_HOSTNAME` is empty, then need to fall back to mariadb host
+			if [ -z "$DB_HOSTNAME" ]; then
+				echo "Warning: No read replica found" >&2
+				DB_HOSTNAME="${!DB_HOST}"
+			fi
+			echo "$DB_HOSTNAME"
+			return
+			;;
+		CONFIG)
+			echo "Error: Can't directly handle '$mtkVariableName' variable -- please do separately" >&2
+			exit 1
+			;;
+		*)
+			echo "Error: Unrecognised MTK variable type: $mtkVariableName" >&2
+			exit 1
+	esac
+
+	##### Fall back to LAGOON_SERVICE_NAME
+	echo "${LAGOON_SERVICE_NAME}_USERNAME"
+}
+
 echo "======================="
 echo "Starting image-builder"
 echo "======================="
@@ -165,6 +257,13 @@ BUILDER_MTK_YAML_BASE64=$(projectEnvironmentVariableCheck BUILDER_MTK_YAML_BASE6
 # optional
 BUILDER_PUSH_TAGS=$(projectEnvironmentVariableCheck BUILDER_PUSH_TAGS "both")
 
+# Set up the MTK variables
+MTK_DUMP_HOSTNAME=$(calc_MTK_variable HOSTNAME "$MTK_DUMP_HOSTNAME")
+MTK_DUMP_DATABASE=$(calc_MTK_variable DATABASE "$MTK_DUMP_DATABASE")
+MTK_DUMP_USERNAME=$(calc_MTK_variable USERNAME "$MTK_DUMP_USERNAME")
+MTK_DUMP_PASSWORD=$(calc_MTK_variable PASSWORD "$MTK_DUMP_PASSWORD")
+
+
 # check the pattern
 if [ -z $BUILDER_BACKUP_IMAGE_NAME ]; then
 	# default pattern to parse if no image provided
@@ -175,7 +274,7 @@ else
 fi
 echo $BUILDER_BACKUP_IMAGE_NAME
 
-# error out if username and password aren't provided
+# error out if registry username and password aren't provided
 if [ -z $BUILDER_REGISTRY_USERNAME ]; then
 	echo "BUILDER_REGISTRY_USERNAME not defined"
 	exit 1
@@ -189,7 +288,6 @@ fi
 # Generic variables
 date=`date -I`
 san_db_dump_filename="sanitised-dump.sql"
-san_db_tarfile="container-${date}.tar"
 backup_image_tag=${BUILDER_BACKUP_IMAGE_TAG:-"backup-${date}"}
 backup_image_full="${BUILDER_BACKUP_IMAGE_NAME}:${backup_image_tag}"
 
@@ -199,30 +297,14 @@ backup_image_full="${BUILDER_BACKUP_IMAGE_NAME}:${backup_image_tag}"
 echo
 echo "=== Phase 2: MTK dump ==="
 
-# set the DB_X variables to what would be the lagoon service variable values, these are handled automatically from the LAGOON_SERVICE_NAME
-# these will use bash references later; eg: ${!DB_HOST}
-DB_HOST="${LAGOON_SERVICE_NAME}_HOST"
-DB_READREPLICA_HOSTS="${LAGOON_SERVICE_NAME}_READREPLICA_HOSTS"
-DB_USERNAME="${LAGOON_SERVICE_NAME}_USERNAME"
-DB_PASSWORD="${LAGOON_SERVICE_NAME}_PASSWORD"
-DB_DATABASE="${LAGOON_SERVICE_NAME}_DATABASE"
-
-# Picks one of the read replicas to use when dumping the database
-DB_HOSTNAME=`echo "${!DB_READREPLICA_HOSTS}" | perl -F, -ane '$index=int(rand(scalar(@F)));print $F[$index]'`
-# if `DB_HOSTNAME` is empty, then need to fall back to mariadb host
-if [ -z $DB_HOSTNAME ]; then
-    DB_HOSTNAME="${!DB_HOST}"
-fi
+export MTK_DUMP_HOSTNAME MTK_DUMP_DATABASE MTK_DUMP_USERNAME MTK_DUMP_PASSWORD
 
 # dump the MTK YAML to the mtk file if it has been provided, otherwise mtk will just dump the entire database as is
-if [ ! -z $BUILDER_MTK_YAML_BASE64 ]; then
-	echo $BUILDER_MTK_YAML_BASE64 | base64 -d > mtk.yml
-	export MTK_DUMP_CONFIG="mtk.yml"
+if [ -n "$BUILDER_MTK_YAML_BASE64" ]; then
+	mtk_filename=mtk.yml
+	echo "$BUILDER_MTK_YAML_BASE64" | base64 -d > $mtk_filename
+	export MTK_DUMP_CONFIG="$mtk_filename"
 fi
-export MTK_DUMP_HOSTNAME="$DB_HOSTNAME"
-export MTK_DUMP_USERNAME="${!DB_USERNAME}"
-export MTK_DUMP_PASSWORD="${!DB_PASSWORD}"
-export MTK_DUMP_DATABASE="${!DB_DATABASE}"
 
 # Run MTK here
 mtk-dump > "$san_db_dump_filename"