added superset-service and Caddyfile for the same

Caddyfile.example

@@ -3,4 +3,5 @@ import ./common/minio/Caddyfile
 import ./common/environment/Caddyfile
 #import ./common/fusionauth/Caddyfile
 # The registry doesn't have a auth thus exposing it publicly means anyone can access the images pushed to this registry
-#import ./common/registry/Caddyfile
+#import ./common/registry/Caddyfile
+#import ./common/superset/Caddyfile

common/superset/Caddyfile

@@ -0,0 +1,3 @@
+{$DOMAIN_SCHEME}://superset.{$DOMAIN_NAME} {
+    reverse_proxy superset:8088
+}

common/superset/docker-compose.yaml

@@ -0,0 +1,43 @@
+services:
+  superset:
+    image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}}
+    profiles: ["devops","superset"]
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: always
+    <<: [*superset-environment , *superset-volumes]
+    depends_on:
+      - db
+      - redis
+      - clickhouse
+      - superset_init
+      - superset_worker
+      - superset_worker_beat
+
+  superset_init:
+    image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}}
+    command: ["/app/docker/docker-init.sh"]
+    user: "root"
+    <<: [*superset-environment ,*superset-depends-on, *superset-volumes]
+
+
+  superset_worker:
+    image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}}
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    restart: always
+    user: "root"
+    <<: [*superset-environment ,*superset-depends-on, *superset-volumes]
+
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
+        ]
+
+  superset_worker_beat:
+    image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}}
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    restart: always
+    user: "root"
+    <<: [*superset-environment ,*superset-depends-on, *superset-volumes]

common/superset/docker/docker-bootstrap.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+fi
+
+#
+# playwright is an optional package - run only if it is installed
+#
+if command -v playwright > /dev/null 2>&1; then
+  playwright install-deps
+  playwright install chromium
+fi
+
+case "${1}" in
+  worker)
+    echo "Starting Celery worker..."
+    celery --app=superset.tasks.celery_app:app worker -O fair -l INFO
+    ;;
+  beat)
+    echo "Starting Celery beat..."
+    rm -f /tmp/celerybeat.pid
+    celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+    ;;
+  app)
+    echo "Starting web app (using development server)..."
+    flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+    ;;
+  app-gunicorn)
+    echo "Starting web app..."
+    /usr/bin/run-server.sh
+    ;;
+  *)
+    echo "Unknown Operation!!!"
+    ;;
+esac

common/superset/docker/docker-ci.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+/app/docker/docker-init.sh
+
+# TODO: copy config overrides from ENV vars
+
+# TODO: run celery in detached state
+export SERVER_THREADS_AMOUNT=8
+# start up the web server
+
+/usr/bin/run-server.sh

common/superset/docker/docker-frontend.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+# Packages needed for puppeteer:
+apt update
+if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then
+    apt install -y chromium
+fi
+
+if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
+    cd /app/superset-frontend
+    npm install -f --no-optional --global webpack webpack-cli
+    npm install -f --no-optional
+
+    echo "Running frontend"
+    npm run dev
+else
+    echo "Skipping frontend build steps - YOU RUN IT MANUALLY ON THE HOST!"
+fi

common/superset/docker/docker-init.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+#
+# Always install local overrides first
+#
+/app/docker/docker-bootstrap.sh
+
+STEP_CNT=4
+
+echo_step() {
+cat <<EOF
+
+######################################################################
+
+
+Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
+
+
+######################################################################
+
+EOF
+}
+
+apt update &&  apt install -yy postgresql-client
+
+PGPASSWORD=${POSTGRES_PASSWORD} psql -h db -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}"  <<-EOSQL
+DO \$\$
+BEGIN
+    CREATE USER superset WITH PASSWORD '${DATABASE_PASSWORD}';
+    EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
+END
+\$\$;
+SELECT 'CREATE DATABASE superset OWNER superset'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'superset')\gexec
+EOSQL
+
+ADMIN_PASSWORD=${SUPERSET_ADMIN_PASSWORD}
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    ADMIN_PASSWORD="general"
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+# Initialize the database
+echo_step "1" "Starting" "Applying DB migrations"
+superset db upgrade
+echo_step "1" "Complete" "Applying DB migrations"
+
+# Create an admin user
+echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
+superset fab create-admin \
+              --username admin \
+              --firstname Superset \
+              --lastname Admin \
+              --email [email protected] \
+              --password $ADMIN_PASSWORD
+echo_step "2" "Complete" "Setting up admin user"
+# Create default roles and permissions
+echo_step "3" "Starting" "Setting up roles and perms"
+superset init
+echo_step "3" "Complete" "Setting up roles and perms"
+
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+    # Load some data to play with
+    echo_step "4" "Starting" "Loading examples"
+    # If Cypress run which consumes superset_test_config – load required data for tests
+    if [ "$CYPRESS_CONFIG" == "true" ]; then
+        superset load_test_users
+        superset load_examples --load-test-data
+    else
+        superset load_examples --force
+    fi
+    echo_step "4" "Complete" "Loading examples"
+fi

common/superset/docker/frontend-mem-nag.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+# We need at least 3GB of free mem...
+MIN_MEM_FREE_GB=3
+MIN_MEM_FREE_KB=$(($MIN_MEM_FREE_GB*1000000))
+
+echo_mem_warn() {
+  MEM_FREE_KB=$(awk '/MemFree/ { printf "%s \n", $2 }' /proc/meminfo)
+  MEM_FREE_GB=$(awk '/MemFree/ { printf "%s \n", $2/1024/1024 }' /proc/meminfo)
+
+  if [[ "${MEM_FREE_KB}" -lt "${MIN_MEM_FREE_KB}" ]]; then
+    cat <<EOF
+    ===============================================
+    ========  Memory Insufficient Warning =========
+    ===============================================
+
+    It looks like you only have ${MEM_FREE_GB}GB of
+    memory free. Please increase your Docker
+    resources to at least ${MIN_MEM_FREE_GB}GB
+
+    ===============================================
+    ========  Memory Insufficient Warning =========
+    ===============================================
+EOF
+  else
+    echo "Memory check Ok [${MEM_FREE_GB}GB free]"
+  fi
+}
+
+# Always nag if they're low on mem...
+echo_mem_warn

common/superset/docker/requirements-local.txt

@@ -0,0 +1 @@
+requirements-local.txt

common/superset/docker/run-server.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+HYPHEN_SYMBOL='-'
+
+gunicorn \
+    --bind "${SUPERSET_BIND_ADDRESS:-0.0.0.0}:${SUPERSET_PORT:-8088}" \
+    --access-logfile "${ACCESS_LOG_FILE:-$HYPHEN_SYMBOL}" \
+    --error-logfile "${ERROR_LOG_FILE:-$HYPHEN_SYMBOL}" \
+    --workers ${SERVER_WORKER_AMOUNT:-1} \
+    --worker-class ${SERVER_WORKER_CLASS:-gthread} \
+    --threads ${SERVER_THREADS_AMOUNT:-20} \
+    --timeout ${GUNICORN_TIMEOUT:-60} \
+    --keep-alive ${GUNICORN_KEEPALIVE:-2} \
+    --max-requests ${WORKER_MAX_REQUESTS:-0} \
+    --max-requests-jitter ${WORKER_MAX_REQUESTS_JITTER:-0} \
+    --limit-request-line ${SERVER_LIMIT_REQUEST_LINE:-0} \
+    --limit-request-field_size ${SERVER_LIMIT_REQUEST_FIELD_SIZE:-0} \
+    "${FLASK_APP}"

docker-compose.yaml.example

@@ -3,9 +3,11 @@ include:
 #  - ./common/minio/docker-compose.yaml
 #  - ./common/environment/docker-compose.yaml
 #  - ./common/fusionauth/docker-compose.yaml
+#  - ./common/superset/docker-compose.yaml
   - ./common/registry/docker-compose.yaml
 
 
+
 services:
   caddy:
     extends: