feat: docker image with both clickhouse and redpandas embedded, runni…

…ng at the same time

apps/create-igloo-app/scripts/release.sh

@@ -1,4 +1,5 @@
 #/usr/bin/env bash
+set -eo pipefail
 
 version=$1
 

apps/create-moose-app/scripts/release.sh

@@ -1,5 +1,7 @@
 #/usr/bin/env bash
 
+set -eo pipefail
+
 version=$1
 
 cd apps/create-moose-app

apps/igloo-cli-npm/scripts/release-bin.sh

@@ -1,5 +1,7 @@
 #/usr/bin/env bash
 
+set -eo pipefail
+
 export node_version=$1
 build_target=$2
 build_os=$3

apps/igloo-cli-npm/scripts/release-cli.sh

@@ -1,5 +1,7 @@
 #/usr/bin/env bash
 
+set -eo pipefail
+
 # This script should be called from the root of the repository
 
 version=$1

apps/moose-cli-npm/scripts/release-bin.sh

@@ -1,5 +1,7 @@
 #/usr/bin/env bash
 
+set -eo pipefail
+
 export node_version=$1
 build_target=$2
 build_os=$3

apps/moose-cli-npm/scripts/release-cli.sh

@@ -1,5 +1,7 @@
 #/usr/bin/env bash
 
+set -eo pipefail
+
 # This script should be called from the root of the repository
 
 version=$1

deploy/docker-fullstack-image/Dockerfile

@@ -0,0 +1,116 @@
+FROM ubuntu:22.04
+
+# see https://github.com/moby/moby/issues/4032#issuecomment-192327844
+ARG DEBIAN_FRONTEND=noninteractive
+
+# ARG for quick switch to a given ubuntu mirror
+ARG apt_archive="http://archive.ubuntu.com"
+RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list 
+
+# USERS Setup
+
+# user/group precreated explicitly with fixed uid/gid on purpose.
+# It is especially important for rootless containers: in that case entrypoint
+# can't do chown and owners of mounted volumes should be configured externally.
+# We do that in advance at the begining of Dockerfile before any packages will be
+# installed to prevent picking those uid / gid by some unrelated software.
+# The same uid / gid (101) is used both for alpine and ubuntu.
+RUN groupadd -r redpanda --gid=101 \
+ && useradd -r -g redpanda --uid=101 --home-dir=/var/lib/redpanda --shell=/bin/bash redpanda
+
+RUN groupadd -r clickhouse --gid=102 \
+ && useradd -r -g clickhouse --uid=102 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse
+
+# Some common dependencies
+RUN apt-get update \
+ && apt-get upgrade -yq \
+ && apt-get install --yes --no-install-recommends \
+ ca-certificates \
+ locales \
+ tzdata \
+ wget \
+ curl \
+ supervisor \
+ apt-transport-https \
+ dirmngr \
+ gnupg2 \
+ && apt-get clean \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /var/cache/debconf \
+ /tmp/*
+
+RUN mkdir -p /var/log/supervisor
+
+# Install Redpandas
+# TODO pin version
+
+RUN curl -1sLf 'https://dl.redpanda.com/nzc4ZYQK3WRGd9sy/redpanda/cfg/setup/bash.deb.sh' | bash && apt install redpanda -y
+
+RUN curl -1sLf 'https://dl.redpanda.com/nzc4ZYQK3WRGd9sy/redpanda/cfg/setup/bash.deb.sh' | bash && apt install redpanda-console -y
+
+RUN rpk redpanda mode production && \ 
+ rpk redpanda config bootstrap --self 127.0.0.1 --ips 127.0.0.1 && \
+ rpk redpanda config set redpanda.empty_seed_starts_cluster false
+
+RUN mkdir -p /var/lib/redpanda/data
+
+VOLUME /var/lib/redpanda
+
+# Install Clickhouse
+# Steps taken from https://github.com/ClickHouse/ClickHouse/blob/master/docker/server/Dockerfile.ubuntu
+
+ARG CLICKHOUSE_REPO_CHANNEL="stable"
+ARG CLICKHOUSE_REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${CLICKHOUSE_REPO_CHANNEL} main"
+ARG CLICKHOUSE_VERSION="23.12.2.59"
+ARG CLICKHOUSE_PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
+
+RUN mkdir -p /etc/apt/sources.list.d \
+ && GNUPGHOME=$(mktemp -d) \
+ && GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring \
+ --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
+ --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 \
+ && rm -rf "$GNUPGHOME" \
+ && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
+ && echo "${CLICKHOUSE_REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
+ && echo "installing from repository: ${CLICKHOUSE_REPOSITORY}" \
+ && apt-get update \
+ && for package in ${CLICKHOUSE_PACKAGES}; do \
+ packages="${packages} ${package}=${CLICKHOUSE_VERSION}" \
+ ; done \
+ && apt-get install --allow-unauthenticated --yes --no-install-recommends ${packages} || exit 1 \
+ && rm -rf \
+ /var/lib/apt/lists/* \
+ /var/cache/debconf \
+ /tmp/* \
+ && apt-get autoremove --purge -yq libksba8 \
+ && apt-get autoremove -yq
+
+# post install
+# we need to allow "others" access to clickhouse folder, because docker container
+# can be started with arbitrary uid (openshift usecase)
+RUN clickhouse-local -q 'SELECT * FROM system.build_options' \
+ && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client \
+ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client
+
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+ENV TZ UTC
+
+RUN mkdir /docker-entrypoint-initdb.d
+
+COPY clickhouse-config.xml /etc/clickhouse-server/config.d/
+COPY run-clickhouse.sh /run-clickhouse.sh
+RUN chmod +x /run-clickhouse.sh
+
+VOLUME /var/lib/clickhouse
+
+# Install Moose
+
+
+# Run it all
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+CMD ["/usr/bin/supervisord"]

deploy/docker-fullstack-image/README.md

@@ -0,0 +1,11 @@
+## Build
+
+```
+docker build .
+```
+
+## Run
+
+```
+docker run --memory=4g --volume /location/on/disk:/var/lib/redpanda/data <sha>
+```

deploy/docker-fullstack-image/clickhouse-config.xml

@@ -0,0 +1,12 @@
+<clickhouse>
+ <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
+ <listen_host>::</listen_host>
+ <listen_host>0.0.0.0</listen_host>
+ <listen_try>1</listen_try>
+
+ <!--
+ <logger>
+ <console>1</console>
+ </logger>
+ -->
+</clickhouse>

deploy/docker-fullstack-image/run-clickhouse.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+
+# Copied from https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/docker/server/entrypoint.sh
+
+set -eo pipefail
+shopt -s nullglob
+
+DO_CHOWN=1
+if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then
+ DO_CHOWN=0
+fi
+
+CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}"
+CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}"
+
+# support --user
+if [ "$(id -u)" = "0" ]; then
+ USER=$CLICKHOUSE_UID
+ GROUP=$CLICKHOUSE_GID
+else
+ USER="$(id -u)"
+ GROUP="$(id -g)"
+ DO_CHOWN=0
+fi
+
+# set some vars
+CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}"
+
+# get CH directories locations
+DATA_DIR="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=path || true)"
+TMP_DIR="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=tmp_path || true)"
+USER_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=user_files_path || true)"
+LOG_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=logger.log || true)"
+LOG_DIR=""
+if [ -n "$LOG_PATH" ]; then LOG_DIR="$(dirname "$LOG_PATH")"; fi
+ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=logger.errorlog || true)"
+ERROR_LOG_DIR=""
+if [ -n "$ERROR_LOG_PATH" ]; then ERROR_LOG_DIR="$(dirname "$ERROR_LOG_PATH")"; fi
+FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=format_schema_path || true)"
+
+# There could be many disks declared in config
+readarray -t DISKS_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.path' || true)
+readarray -t DISKS_METADATA_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.metadata_path' || true)
+
+CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
+CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
+CLICKHOUSE_DB="${CLICKHOUSE_DB:-}"
+CLICKHOUSE_ACCESS_MANAGEMENT="${CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT:-0}"
+
+for dir in "$DATA_DIR" \
+ "$ERROR_LOG_DIR" \
+ "$LOG_DIR" \
+ "$TMP_DIR" \
+ "$USER_PATH" \
+ "$FORMAT_SCHEMA_PATH" \
+ "${DISKS_PATHS[@]}" \
+ "${DISKS_METADATA_PATHS[@]}"
+do
+ # check if variable not empty
+ [ -z "$dir" ] && continue
+ # ensure directories exist
+ if [ "$DO_CHOWN" = "1" ]; then
+ mkdir="mkdir"
+ else
+ # if DO_CHOWN=0 it means that the system does not map root user to "admin" permissions
+ # it mainly happens on NFS mounts where root==nobody for security reasons
+ # thus mkdir MUST run with user id/gid and not from nobody that has zero permissions
+ mkdir="/usr/bin/clickhouse su "${USER}:${GROUP}" mkdir"
+ fi
+ if ! $mkdir -p "$dir"; then
+ echo "Couldn't create necessary directory: $dir"
+ exit 1
+ fi
+
+ if [ "$DO_CHOWN" = "1" ]; then
+ # ensure proper directories permissions
+ # but skip it for if directory already has proper premissions, cause recursive chown may be slow
+ if [ "$(stat -c %u "$dir")" != "$USER" ] || [ "$(stat -c %g "$dir")" != "$GROUP" ]; then
+ chown -R "$USER:$GROUP" "$dir"
+ fi
+ fi
+done
+
+# if clickhouse user is defined - create it (user "default" already exists out of box)
+if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ] || [ "$CLICKHOUSE_ACCESS_MANAGEMENT" != "0" ]; then
+ echo "$0: create new user '$CLICKHOUSE_USER' instead 'default'"
+ cat <<EOT > /etc/clickhouse-server/users.d/default-user.xml
+ <clickhouse>
+ <!-- Docs: <https://clickhouse.com/docs/en/operations/settings/settings_users/> -->
+ <users>
+ <!-- Remove default user -->
+ <default remove="remove">
+ </default>
+
+ <${CLICKHOUSE_USER}>
+ <profile>default</profile>
+ <networks>
+ <ip>::/0</ip>
+ </networks>
+ <password>${CLICKHOUSE_PASSWORD}</password>
+ <quota>default</quota>
+ <access_management>${CLICKHOUSE_ACCESS_MANAGEMENT}</access_management>
+ </${CLICKHOUSE_USER}>
+ </users>
+ </clickhouse>
+EOT
+fi
+
+# checking $DATA_DIR for initialization
+if [ -d "${DATA_DIR%/}/data" ]; then
+ DATABASE_ALREADY_EXISTS='true'
+fi
+
+# only run initialization on an empty data directory
+if [ -z "${DATABASE_ALREADY_EXISTS}" ]; then
+ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then
+ # port is needed to check if clickhouse-server is ready for connections
+ HTTP_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=http_port --try)"
+ HTTPS_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=https_port --try)"
+
+ if [ -n "$HTTP_PORT" ]; then
+ URL="http://127.0.0.1:$HTTP_PORT/ping"
+ else
+ URL="https://127.0.0.1:$HTTPS_PORT/ping"
+ fi
+
+ # Listen only on localhost until the initialization is done
+ /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 &
+ pid="$!"
+
+ # check if clickhouse is ready to accept connections
+ # will try to send ping clickhouse via http_port (max 1000 retries by default, with 1 sec timeout and 1 sec delay between retries)
+ tries=${CLICKHOUSE_INIT_TIMEOUT:-1000}
+ while ! wget --spider --no-check-certificate -T 1 -q "$URL" 2>/dev/null; do
+ if [ "$tries" -le "0" ]; then
+ echo >&2 'ClickHouse init process failed.'
+ exit 1
+ fi
+ tries=$(( tries-1 ))
+ sleep 1
+ done
+
+ clickhouseclient=( clickhouse-client --multiquery --host "127.0.0.1" -u "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" )
+
+ echo
+
+ # create default database, if defined
+ if [ -n "$CLICKHOUSE_DB" ]; then
+ echo "$0: create database '$CLICKHOUSE_DB'"
+ "${clickhouseclient[@]}" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB";
+ fi
+
+ for f in /docker-entrypoint-initdb.d/*; do
+ case "$f" in
+ *.sh)
+ if [ -x "$f" ]; then
+ echo "$0: running $f"
+ "$f"
+ else
+ echo "$0: sourcing $f"
+ # shellcheck source=/dev/null
+ . "$f"
+ fi
+ ;;
+ *.sql) echo "$0: running $f"; "${clickhouseclient[@]}" < "$f" ; echo ;;
+ *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${clickhouseclient[@]}"; echo ;;
+ *) echo "$0: ignoring $f" ;;
+ esac
+ echo
+ done
+
+ if ! kill -s TERM "$pid" || ! wait "$pid"; then
+ echo >&2 'Finishing of ClickHouse init process failed.'
+ exit 1
+ fi
+ fi
+else
+ echo "ClickHouse Database directory appears to contain a database; Skipping initialization"
+fi
+
+# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments
+if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
+ # Watchdog is launched by default, but does not send SIGINT to the main process,
+ # so the container can't be finished by ctrl+c
+ CLICKHOUSE_WATCHDOG_ENABLE=${CLICKHOUSE_WATCHDOG_ENABLE:-0}
+ export CLICKHOUSE_WATCHDOG_ENABLE
+
+ # An option for easy restarting and replacing clickhouse-server in a container, especially in Kubernetes.
+ # For example, you can replace the clickhouse-server binary to another and restart it while keeping the container running.
+ if [[ "${CLICKHOUSE_DOCKER_RESTART_ON_EXIT:-0}" -eq "1" ]]; then
+ while true; do
+ # This runs the server as a child process of the shell script:
+ /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" ||:
+ echo >&2 'ClickHouse Server exited, and the environment variable CLICKHOUSE_DOCKER_RESTART_ON_EXIT is set to 1. Restarting the server.'
+ done
+ else
+ # This replaces the shell script with the server:
+ exec /usr/bin/clickhouse su "${USER}:${GROUP}" /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@"
+ fi
+fi
+
+# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image
+exec "$@"