Skip to content

Commit

Permalink
chore: adjust image startup scripts, set chart branch
Browse files Browse the repository at this point in the history
  • Loading branch information
shreddedbacon committed Dec 10, 2024
1 parent d60014a commit e99f499
Show file tree
Hide file tree
Showing 5 changed files with 323 additions and 1 deletion.
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ STERN_VERSION = v2.6.1
CHART_TESTING_VERSION = v3.11.0
K3D_IMAGE = docker.io/rancher/k3s:v1.31.0-k3s1
TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
CHARTS_TREEISH = main
CHARTS_TREEISH = mysql-image-support
TASK_IMAGES = task-activestandby

# the name of the docker network to create
Expand Down Expand Up @@ -580,6 +580,7 @@ k3d/test: k3d/setup
OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
SKIP_ALL_DEPS=true \
CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
USE_CALICO_CNI=false \
LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
Expand Down Expand Up @@ -614,6 +615,7 @@ k3d/setup: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build
OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
SKIP_INSTALL_REGISTRY=true \
CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
USE_CALICO_CNI=false \
LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
Expand Down Expand Up @@ -673,6 +675,7 @@ k3d/dev: build
&& $(MAKE) install-lagoon-core DOCKER_NETWORK=$(DOCKER_NETWORK) IMAGE_TAG=$(SAFE_BRANCH_NAME) DISABLE_CORE_HARBOR=true \
HELM=$(HELM) KUBECTL=$(KUBECTL) \
JQ=$(JQ) \
CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
OVERRIDE_BUILD_DEPLOY_DIND_IMAGE=uselagoon/build-deploy-image:${BUILD_DEPLOY_IMAGE_TAG} \
$$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGETAG)') \
$$([ $(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY) ] && echo 'OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY=$(OVERRIDE_BUILD_DEPLOY_CONTROLLER_IMAGE_REPOSITORY)') \
Expand Down Expand Up @@ -797,6 +800,7 @@ k3d/retest:
OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
SKIP_ALL_DEPS=true \
CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
USE_CALICO_CNI=false \
LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
Expand Down
2 changes: 2 additions & 0 deletions services/api-db/Dockerfile.mysql
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
RUN chown -R mysql /legacy-migration-scripts/ \
&& /bin/fix-permissions /legacy-migration-scripts/
COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
USER mysql
ENV MYSQL_DATABASE=infrastructure \
Expand Down
156 changes: 156 additions & 0 deletions services/api-db/mysql-init.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
#!/usr/bin/env bash

set -eo pipefail

if [ "$(ls -A /etc/mysql/conf.d/)" ]; then
ep /etc/mysql/conf.d/*
fi

if [ "${1:0:1}" = '-' ]; then
set -- mysqld "$@"
fi

wantHelp=
for arg; do
case "$arg" in
-'?'|--help|--print-defaults|-V|--version)
wantHelp=1
break
;;
esac
done

# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder
# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container).
# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting
# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system
# the data will be prefilled and persistent across container restarts.
if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then
if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying"
else
echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'"
CUR_DIR=${PWD}
cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -)
cd $CUR_DIR
fi
fi

ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf

if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
if [ ! -d "/run/mysqld" ]; then
mkdir -p /var/run/mysqld
chown -R mysql:mysql /var/run/mysqld
fi

MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30}
MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1}

if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
echo "MySQL directory already present, skipping creation"

echo "starting mysql for mysql upgrade."
/usr/sbin/mysqld --skip-networking &
pid="$!"
echo "pid is $pid"

for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
if echo 'SELECT 1' | mysql -u root; then
break
fi
echo 'MySQL init process in progress...'
sleep $MYSQL_INIT_PERIOD_SECONDS
done

if ! kill -s TERM "$pid" || ! wait "$pid"; then
echo >&2 'MySQL init process failed.'
exit 1
fi
else
echo "MySQL data directory not found, creating initial DBs"

/usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr

echo "starting mysql for initdb.d import."
/usr/sbin/mysqld --skip-networking &
pid="$!"
echo "pid is $pid"

for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
if echo 'SELECT 1' | mysql -u root; then
break
fi
echo 'MySQL init process in progress...'
sleep $MYSQL_INIT_PERIOD_SECONDS
done

if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then
MYSQL_ROOT_PASSWORD=`pwgen 16 1`
echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD"
fi

MYSQL_DATABASE=${MYSQL_DATABASE:-""}
MYSQL_USER=${MYSQL_USER:-""}
MYSQL_PASSWORD=${MYSQL_PASSWORD:-""}

tfile=`mktemp`
if [ ! -f "$tfile" ]; then
return 1
fi

cat << EOF > $tfile
DROP DATABASE IF EXISTS test;
USE mysql;
ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD';
DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
FLUSH PRIVILEGES;
EOF

if [ "$MYSQL_DATABASE" != "" ]; then
echo "[i] Creating database: $MYSQL_DATABASE"
echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile
if [ "$MYSQL_USER" != "" ]; then
echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD"
echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile
echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile
fi
fi


cat $tfile
cat $tfile | mysql -v -u root
rm -v -f $tfile

echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "password=${MYSQL_ROOT_PASSWORD}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf

for f in /docker-entrypoint-initdb.d/*; do
if [ -e "$f" ]; then
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql) echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;;
*) echo "$0: ignoring $f" ;;
esac
fi
done

if ! kill -s TERM "$pid" || ! wait "$pid"; then
echo >&2 'MySQL init process failed.'
exit 1
fi

fi

echo "done, now starting daemon"
touch /tmp/startup-init-complete
touch /tmp/mysql-init-complete

fi
4 changes: 4 additions & 0 deletions services/keycloak-db/Dockerfile.mysql
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ ENV MYSQL_DATABASE=keycloak \
# MYSQL_CHARSET=utf8 \
# MYSQL_COLLATION=utf8_general_ci

USER root
COPY mysql-init.bash /lagoon/entrypoints/9999-mysql-init.bash
USER mysql

# not used in mysql8
# COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf
# USER root
Expand Down
156 changes: 156 additions & 0 deletions services/keycloak-db/mysql-init.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
#!/usr/bin/env bash

set -eo pipefail

if [ "$(ls -A /etc/mysql/conf.d/)" ]; then
ep /etc/mysql/conf.d/*
fi

if [ "${1:0:1}" = '-' ]; then
set -- mysqld "$@"
fi

wantHelp=
for arg; do
case "$arg" in
-'?'|--help|--print-defaults|-V|--version)
wantHelp=1
break
;;
esac
done

# check if MYSQL_COPY_DATA_DIR_SOURCE is set, if yes we're coping the contents of the given folder into the data dir folder
# this allows to prefill the datadir with a provided datadir (either added in a Dockerfile build, or mounted into the running container).
# This is different than just setting $MYSQL_DATA_DIR to the source folder, as only /var/lib/mysql is a persistent folder, so setting
# $MYSQL_DATA_DIR to another folder will make mysql to not store the datadir across container restarts, while with this copy system
# the data will be prefilled and persistent across container restarts.
if [ -n "$MYSQL_COPY_DATA_DIR_SOURCE" ]; then
if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
echo "MYSQL_COPY_DATA_DIR_SOURCE is set, but MySQL directory already present in '${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql' skipping copying"
else
echo "MYSQL_COPY_DATA_DIR_SOURCE is set, copying datadir contents from '$MYSQL_COPY_DATA_DIR_SOURCE' to '${MYSQL_DATA_DIR:-/var/lib/mysql}'"
CUR_DIR=${PWD}
cd ${MYSQL_COPY_DATA_DIR_SOURCE}/; tar cf - . | (cd ${MYSQL_DATA_DIR:-/var/lib/mysql}; tar xvf -)
cd $CUR_DIR
fi
fi

ln -sf ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf /home/.my.cnf

if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
if [ ! -d "/run/mysqld" ]; then
mkdir -p /var/run/mysqld
chown -R mysql:mysql /var/run/mysqld
fi

MYSQL_INIT_WAIT_SECONDS=${MYSQL_INIT_WAIT_SECONDS:-30}
MYSQL_INIT_PERIOD_SECONDS=${MYSQL_INIT_PERIOD_SECONDS:-1}

if [ -d ${MYSQL_DATA_DIR:-/var/lib/mysql}/mysql ]; then
echo "MySQL directory already present, skipping creation"

echo "starting mysql for mysql upgrade."
/usr/sbin/mysqld --skip-networking &
pid="$!"
echo "pid is $pid"

for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
if echo 'SELECT 1' | mysql -u root; then
break
fi
echo 'MySQL init process in progress...'
sleep $MYSQL_INIT_PERIOD_SECONDS
done

if ! kill -s TERM "$pid" || ! wait "$pid"; then
echo >&2 'MySQL init process failed.'
exit 1
fi
else
echo "MySQL data directory not found, creating initial DBs"

/usr/sbin/mysqld --initialize-insecure --skip-name-resolve --datadir=${MYSQL_DATA_DIR:-/var/lib/mysql} --basedir=/usr

echo "starting mysql for initdb.d import."
/usr/sbin/mysqld --skip-networking &
pid="$!"
echo "pid is $pid"

for i in $(seq 0 $MYSQL_INIT_WAIT_SECONDS); do
if echo 'SELECT 1' | mysql -u root; then
break
fi
echo 'MySQL init process in progress...'
sleep $MYSQL_INIT_PERIOD_SECONDS
done

if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then
MYSQL_ROOT_PASSWORD=`pwgen 16 1`
echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD"
fi

MYSQL_DATABASE=${MYSQL_DATABASE:-""}
MYSQL_USER=${MYSQL_USER:-""}
MYSQL_PASSWORD=${MYSQL_PASSWORD:-""}

tfile=`mktemp`
if [ ! -f "$tfile" ]; then
return 1
fi

cat << EOF > $tfile
DROP DATABASE IF EXISTS test;
USE mysql;
ALTER USER 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD';
DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
FLUSH PRIVILEGES;
EOF

if [ "$MYSQL_DATABASE" != "" ]; then
echo "[i] Creating database: $MYSQL_DATABASE"
echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> $tfile
if [ "$MYSQL_USER" != "" ]; then
echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD"
echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile
echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%';" >> $tfile
fi
fi


cat $tfile
cat $tfile | mysql -v -u root
rm -v -f $tfile

echo "[client]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "user=root" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "password=${MYSQL_ROOT_PASSWORD}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "[mysql]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "database=${MYSQL_DATABASE}" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "[mysqld]" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf
echo "socket=/run/mysqld/mysqld.sock" >> ${MYSQL_DATA_DIR:-/var/lib/mysql}/.my.cnf

for f in /docker-entrypoint-initdb.d/*; do
if [ -e "$f" ]; then
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql) echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MYSQL_ROOT_PASSWORD}; echo ;;
*) echo "$0: ignoring $f" ;;
esac
fi
done

if ! kill -s TERM "$pid" || ! wait "$pid"; then
echo >&2 'MySQL init process failed.'
exit 1
fi

fi

echo "done, now starting daemon"
touch /tmp/startup-init-complete
touch /tmp/mysql-init-complete

fi

0 comments on commit e99f499

Please sign in to comment.