diff --git a/README.md b/README.md index 728c61f..f32ace0 100644 --- a/README.md +++ b/README.md @@ -13,4 +13,19 @@ The mtk config file should be provided as a base64 encoded string, this allows f ``` $ cat example.mtk.yml | base64 | tr -d '\n\r' cmV3cml0ZToKICAjIERydXBhbCA4CiAgdXNlcnNfZmllbGRfZGF0YToKICAgIG1haWw6IGNvbmNhdCh1aWQsICJAU0FOSVRJU0VEIikKICAgIHBhc3M6ICciU0FOSVRJWkVEX1BBU1NXT1JEIicKICAgICMgV2UgZG9uJ3QgYWxsb3cgbm8gZGlydHkgbmFtZXMgYXJvdW5kIGhlcmUhCiAgICBuYW1lOiBjb25jYXQodWlkLCAiLVNBTklUSVNFRCIpCiAgICBpbml0OiAnIlNBTklUSVNFRF9JTklUIicKICAjIERydXBhbCA3CiAgdXNlcnM6CiAgICBtYWlsOiBjb25jYXQodWlkLCAiQFNBTklUSVNFRCIpCiAgICBwYXNzOiAnIlNBTklUSVpFRF9QQVNTV09SRCInCgp3aGVyZToKICAjIE9ubHkgaW5jbHVkZSBib2R5IGZpZWxkIGRhdGEgZm9yIGN1cnJlbnQgcmV2aXNpb25zLgogIG5vZGVfcmV2aXNpb25fX2JvZHk6IHwtCiAgICAgIHJldmlzaW9uX2lkIElOIChTRUxFQ1QgdmlkIEZST00gbm9kZSkKCm5vZGF0YToKICAtIGNhY2hlKgogIC0gY2FwdGNoYV9zZXNzaW9ucwogIC0gaGlzdG9yeQogIC0gZmxvb2QKICAtIGJhdGNoCiAgLSBxdWV1ZQogIC0gc2Vzc2lvbnMKICAtIHNlbWFwaG9yZQogIC0gc2VhcmNoX2FwaV90YXNrCiAgLSBzZWFyY2hfZGF0YXNldAogIC0gc2VhcmNoX2luZGV4CiAgLSBzZWFyY2hfdG90YWwKICAtIHdhdGNoZG9nCiAgLSB3ZWJmb3JtX3N1Ym1pc3Npb25fZGF0YQoKaWdub3JlOgogIC0gX19BQ1FVSUFfTU9OSVRPUklOR19fCgo= -``` \ No newline at end of file +``` + +## Variables + +An example of the GraphQL needed to create an advanced task is available in +createDumpSanitisedDB.gql. The variables which can be passed in are also +documented in this file (in the "displayName" field of each entry). + +## Process + +The process goes through X basic stages: + +1. Set up all the initial variables +2. MTK creates a database dump that's basically a sanitised .sql file +3. Make docker-style container with sanitised DB (using podman) +4. Save new container to registry diff --git a/createDumpSanitisedDB.gql b/createDumpSanitisedDB.gql index 18d5251..217b8fc 100644 --- a/createDumpSanitisedDB.gql +++ b/createDumpSanitisedDB.gql @@ -79,4 +79,4 @@ mutation createDumpSanitisedDB { } } } -} \ No newline at end of file +} diff --git a/createDumpSanitisedDB_setDBVariables.gql b/createDumpSanitisedDB_setDBVariables.gql new file mode 100644 index 0000000..ea336ab --- /dev/null +++ b/createDumpSanitisedDB_setDBVariables.gql @@ -0,0 +1,134 @@ +mutation createDumpSanitisedDB { + addAdvancedTaskDefinition( + input:{ + name: "Sanitised Database Image Build" + description: "Test making a sanitised database dump image" + confirmationText: "Are you sure you wish to create a sanitised database dump now?" + type: IMAGE + permission: MAINTAINER + image: "uselagoon/database-image-task:v0.0.1" + project: 123 + advancedTaskDefinitionArguments: [ + { + name: "BUILDER_DOCKER_COMPOSE_SERVICE_NAME", + displayName: "OPTIONAL: The name of the docker-compose service to backup (defaults to mariadb)", + type: STRING + optional: true + }, + { + name: "BUILDER_BACKUP_IMAGE_NAME", + displayName: "The name of the resulting image to build without tag (eg, for dockerhub myproject/image, or for custom registry quay.io/myproject/image, can also be a pattern like '${registry}/${organization}/${project}/${environment}/${service}-data')", + type: STRING + # defaultValue: "${registry}/${organization}/${project}/${environment}/${service}-data" + }, + { + name: "BUILDER_IMAGE_NAME", + displayName: "OPTIONAL: The name of the builder source image to do the initial db import (defaults to mariadb:10.6)", + type: STRING + optional: true + }, + { + name: "BUILDER_CLEAN_IMAGE_NAME", + displayName: "OPTIONAL: The name of the clean source image that will be built into (defaults to uselagoon/mariadb-10.6-drupal:latest)", + type: STRING + optional: true + }, + { + name: "BUILDER_MTK_YAML_BASE64", + displayName: "OPTIONAL: The base64 encoded value of the mtk dump file to use (defaults to no config)", + type: STRING + optional: true + }, + { + name: "BUILDER_REGISTRY_USERNAME", + displayName: "The username to log in to registry with", + type: STRING + # defaultValue: "username" + }, + { + name: "BUILDER_REGISTRY_PASSWORD", + displayName: "The password to log in to registry with", + type: STRING + # defaultValue: "password" + }, + { + name: "BUILDER_REGISTRY_HOST", + displayName: "OPTIONAL: If not using dockerhub, define the registry to use (eg quay.io)", + type: STRING + optional: true + }, + { + name: "BUILDER_REGISTRY_ORGANIZATION", + displayName: "OPTIONAL: if you want to provide an organization base for the backup image name to parse", + type: STRING + optional: true + }, + # Database hostname + { + name: "BUILDER_MTK_DUMP_HOSTNAME", + displayName: "OPTIONAL: The database hostname you want to use", + type: STRING + optional: true + }, + { + name: "BUILDER_MTK_DUMP_HOSTNAME_NAME", + displayName: "OPTIONAL: The name of the variable that contains the database hostname you want to use", + type: STRING + optional: true + }, + # Database name + { + name: "BUILDER_MTK_DUMP_DATABASE", + displayName: "OPTIONAL: The database name you want to use", + type: STRING + optional: true + }, + { + name: "BUILDER_MTK_DUMP_DATABASE_NAME", + displayName: "OPTIONAL: The name of the variable that contains the database name you want to use", + type: STRING + optional: true + }, + # Database username + { + name: "BUILDER_MTK_DUMP_USERNAME", + displayName: "OPTIONAL: The database username you want to use", + type: STRING + optional: true + }, + { + name: "BUILDER_MTK_DUMP_USERNAME_NAME", + displayName: "OPTIONAL: The name of the variable that contains the database username you want to use", + type: STRING + optional: true + }, + # Database password + { + name: "BUILDER_MTK_DUMP_PASSWORD", + displayName: "OPTIONAL: The database password you want to use", + type: STRING + optional: true + }, + { + name: "BUILDER_MTK_DUMP_PASSWORD_NAME", + displayName: "OPTIONAL: The name of the variable that contains the database password you want to use", + type: STRING + optional: true + } + ] + } + ){ + ... on AdvancedTaskDefinitionImage { + id + name + description + image + confirmationText + advancedTaskDefinitionArguments { + type + name + displayName + } + } + } +} diff --git a/mariadb-image-builder b/mariadb-image-builder index 461748f..1fccb9b 100644 --- a/mariadb-image-builder +++ b/mariadb-image-builder @@ -129,6 +129,98 @@ function projectEnvironmentVariableCheck() { echo "$existingValue" } +# function to return (echo) one of the MTK environment variables. Order of things to check: +# - Existing value +# - Passed variable name +# - Created from LAGOON_SERVICE_NAME +# +# Parameters are: +# - Name of MTK variable to set +# - Search key to use when searching JSON +# +# Also reads: +# Variable | Check +# ---------------------------------------------------+-------------------------------- +# BUILDER_MTK_DUMP_ | projectEnvironmentVariableCheck +# BUILDER_MTK_DUMP__NAME | projectEnvironmentVariableCheck +# The variable named in BUILDER_MTK_DUMP__NAME | No +# LAGOON_SERVICE_NAME | Should already be checked +function calc_MTK_variable() { + local mtkVariableName="$1" + local jsonSearchKey="$2" + + # check for argument + [ "$mtkVariableName" ] || return + + ##### If MTK_DUMP_* has been set + # Example values + # If MTK_DUMP_DATABASE = foodb + # $final_variable_name = MTK_DUMP_DATABASE + # ${!final_variable_name} = foodb + local final_variable_name="MTK_DUMP_${mtkVariableName}" + local source_variable_name="BUILDER_${final_variable_name}" + declare "$source_variable_name"=$(projectEnvironmentVariableCheck "$source_variable_name" "${!source_variable_name}" "$jsonSearchKey") + + if [ -n "${!source_variable_name}" ]; then + echo "${!source_variable_name}" + return + fi + + ##### If MTK_DUMP_*_NAME has been set + # Example values + # If DB_NAME_CENTRAL=foodb and MTK_DUMP_DATABASE_NAME = DB_NAME_CENTRAL + # $nameholder_variable_name = MTK_DUMP_DATABASE_NAME + # $use_var = DB_NAME_CENTRAL + # ${!use_var} = foodb + local nameholder_variable_name="${final_variable_name}_NAME" + declare "$nameholder_variable_name"=$(projectEnvironmentVariableCheck "$nameholder_variable_name" "${!nameholder_variable_name}") + + if [ -n "${!nameholder_variable_name}" ]; then + local use_var=${!nameholder_variable_name} + if [ "${!use_var}" ]; then + echo "${!use_var}" + return + else + echo "Warning: Could not find value for $nameholder_variable_name > $use_var" >&2 + fi + fi + + ##### Special settings for special types + case $mtkVariableName in + DATABASE | USERNAME | PASSWORD) + # Do nothing + ;; + HOSTNAME) + # Set the DB_X host variables to what would be the lagoon service variable values. + # These are generated automatically from the LAGOON_SERVICE_NAME + # these will use bash references later; eg: ${!DB_HOST} + local DB_HOST="${LAGOON_SERVICE_NAME}_HOST" + local DB_READREPLICA_HOSTS="${LAGOON_SERVICE_NAME}_READREPLICA_HOSTS" + + # Picks one of the read replicas to use when dumping the database + local DB_HOSTNAME + DB_HOSTNAME=$(echo "${!DB_READREPLICA_HOSTS}" | perl -F, -ane '$index=int(rand(scalar(@F)));print $F[$index]') + # if `DB_HOSTNAME` is empty, then need to fall back to mariadb host + if [ -z "$DB_HOSTNAME" ]; then + echo "Warning: No read replica found" >&2 + DB_HOSTNAME="${!DB_HOST}" + fi + echo "$DB_HOSTNAME" + return + ;; + CONFIG) + echo "Error: Can't directly handle '$mtkVariableName' variable -- please do separately" >&2 + exit 1 + ;; + *) + echo "Error: Unrecognised MTK variable type: $mtkVariableName" >&2 + exit 1 + esac + + ##### Fall back to LAGOON_SERVICE_NAME + echo "${LAGOON_SERVICE_NAME}_USERNAME" +} + echo "=======================" echo "Starting image-builder" echo "=======================" @@ -165,6 +257,13 @@ BUILDER_MTK_YAML_BASE64=$(projectEnvironmentVariableCheck BUILDER_MTK_YAML_BASE6 # optional BUILDER_PUSH_TAGS=$(projectEnvironmentVariableCheck BUILDER_PUSH_TAGS "both") +# Set up the MTK variables +MTK_DUMP_HOSTNAME=$(calc_MTK_variable HOSTNAME "$MTK_DUMP_HOSTNAME") +MTK_DUMP_DATABASE=$(calc_MTK_variable DATABASE "$MTK_DUMP_DATABASE") +MTK_DUMP_USERNAME=$(calc_MTK_variable USERNAME "$MTK_DUMP_USERNAME") +MTK_DUMP_PASSWORD=$(calc_MTK_variable PASSWORD "$MTK_DUMP_PASSWORD") + + # check the pattern if [ -z $BUILDER_BACKUP_IMAGE_NAME ]; then # default pattern to parse if no image provided @@ -175,7 +274,7 @@ else fi echo $BUILDER_BACKUP_IMAGE_NAME -# error out if username and password aren't provided +# error out if registry username and password aren't provided if [ -z $BUILDER_REGISTRY_USERNAME ]; then echo "BUILDER_REGISTRY_USERNAME not defined" exit 1 @@ -189,7 +288,6 @@ fi # Generic variables date=`date -I` san_db_dump_filename="sanitised-dump.sql" -san_db_tarfile="container-${date}.tar" backup_image_tag=${BUILDER_BACKUP_IMAGE_TAG:-"backup-${date}"} backup_image_full="${BUILDER_BACKUP_IMAGE_NAME}:${backup_image_tag}" @@ -199,30 +297,14 @@ backup_image_full="${BUILDER_BACKUP_IMAGE_NAME}:${backup_image_tag}" echo echo "=== Phase 2: MTK dump ===" -# set the DB_X variables to what would be the lagoon service variable values, these are handled automatically from the LAGOON_SERVICE_NAME -# these will use bash references later; eg: ${!DB_HOST} -DB_HOST="${LAGOON_SERVICE_NAME}_HOST" -DB_READREPLICA_HOSTS="${LAGOON_SERVICE_NAME}_READREPLICA_HOSTS" -DB_USERNAME="${LAGOON_SERVICE_NAME}_USERNAME" -DB_PASSWORD="${LAGOON_SERVICE_NAME}_PASSWORD" -DB_DATABASE="${LAGOON_SERVICE_NAME}_DATABASE" - -# Picks one of the read replicas to use when dumping the database -DB_HOSTNAME=`echo "${!DB_READREPLICA_HOSTS}" | perl -F, -ane '$index=int(rand(scalar(@F)));print $F[$index]'` -# if `DB_HOSTNAME` is empty, then need to fall back to mariadb host -if [ -z $DB_HOSTNAME ]; then - DB_HOSTNAME="${!DB_HOST}" -fi +export MTK_DUMP_HOSTNAME MTK_DUMP_DATABASE MTK_DUMP_USERNAME MTK_DUMP_PASSWORD # dump the MTK YAML to the mtk file if it has been provided, otherwise mtk will just dump the entire database as is -if [ ! -z $BUILDER_MTK_YAML_BASE64 ]; then - echo $BUILDER_MTK_YAML_BASE64 | base64 -d > mtk.yml - export MTK_DUMP_CONFIG="mtk.yml" +if [ -n "$BUILDER_MTK_YAML_BASE64" ]; then + mtk_filename=mtk.yml + echo "$BUILDER_MTK_YAML_BASE64" | base64 -d > $mtk_filename + export MTK_DUMP_CONFIG="$mtk_filename" fi -export MTK_DUMP_HOSTNAME="$DB_HOSTNAME" -export MTK_DUMP_USERNAME="${!DB_USERNAME}" -export MTK_DUMP_PASSWORD="${!DB_PASSWORD}" -export MTK_DUMP_DATABASE="${!DB_DATABASE}" # Run MTK here mtk-dump > "$san_db_dump_filename"