diff --git a/Caddyfile.example b/Caddyfile.example index e96ebaf..052cdca 100644 --- a/Caddyfile.example +++ b/Caddyfile.example @@ -3,4 +3,5 @@ import ./common/minio/Caddyfile import ./common/environment/Caddyfile #import ./common/fusionauth/Caddyfile # The registry doesn't have a auth thus exposing it publicly means anyone can access the images pushed to this registry -#import ./common/registry/Caddyfile \ No newline at end of file +#import ./common/registry/Caddyfile +#import ./common/superset/Caddyfile diff --git a/common/superset/Caddyfile b/common/superset/Caddyfile new file mode 100644 index 0000000..3cd7f93 --- /dev/null +++ b/common/superset/Caddyfile @@ -0,0 +1,3 @@ +{$DOMAIN_SCHEME}://superset.{$DOMAIN_NAME} { + reverse_proxy superset:8088 +} \ No newline at end of file diff --git a/common/superset/docker-compose.yaml b/common/superset/docker-compose.yaml new file mode 100644 index 0000000..0cdc83f --- /dev/null +++ b/common/superset/docker-compose.yaml @@ -0,0 +1,43 @@ +services: + superset: + image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + profiles: ["devops","superset"] + command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] + user: "root" + restart: always + <<: [*superset-environment , *superset-volumes] + depends_on: + - db + - redis + - clickhouse + - superset_init + - superset_worker + - superset_worker_beat + + superset_init: + image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: ["/app/docker/docker-init.sh"] + user: "root" + <<: [*superset-environment ,*superset-depends-on, *superset-volumes] + + + superset_worker: + image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: ["/app/docker/docker-bootstrap.sh", "worker"] + restart: always + user: "root" + <<: [*superset-environment ,*superset-depends-on, *superset-volumes] + + healthcheck: + test: + [ + "CMD-SHELL", + "celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME", + ] + + superset_worker_beat: + image: ghcr.io/${org}/superset:${SUPERSET_IMAGE_TAG:-${DEFAULT_IMAGE_TAG:?DEFAULT_IMAGE_TAG is not set}} + command: ["/app/docker/docker-bootstrap.sh", "beat"] + restart: always + user: "root" + <<: [*superset-environment ,*superset-depends-on, *superset-volumes] \ No newline at end of file diff --git a/common/superset/docker/docker-bootstrap.sh b/common/superset/docker/docker-bootstrap.sh new file mode 100644 index 0000000..56e6c92 --- /dev/null +++ b/common/superset/docker/docker-bootstrap.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eo pipefail + +REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt" +# If Cypress run – overwrite the password for admin and export env variables +if [ "$CYPRESS_CONFIG" == "true" ]; then + export SUPERSET_CONFIG=tests.integration_tests.superset_test_config + export SUPERSET_TESTENV=true + export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset +fi +# +# Make sure we have dev requirements installed +# +if [ -f "${REQUIREMENTS_LOCAL}" ]; then + echo "Installing local overrides at ${REQUIREMENTS_LOCAL}" + pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}" +else + echo "Skipping local overrides" +fi + +# +# playwright is an optional package - run only if it is installed +# +if command -v playwright > /dev/null 2>&1; then + playwright install-deps + playwright install chromium +fi + +case "${1}" in + worker) + echo "Starting Celery worker..." + celery --app=superset.tasks.celery_app:app worker -O fair -l INFO + ;; + beat) + echo "Starting Celery beat..." + rm -f /tmp/celerybeat.pid + celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule + ;; + app) + echo "Starting web app (using development server)..." + flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 + ;; + app-gunicorn) + echo "Starting web app..." + /usr/bin/run-server.sh + ;; + *) + echo "Unknown Operation!!!" + ;; +esac \ No newline at end of file diff --git a/common/superset/docker/docker-ci.sh b/common/superset/docker/docker-ci.sh new file mode 100644 index 0000000..2114cd2 --- /dev/null +++ b/common/superset/docker/docker-ci.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +/app/docker/docker-init.sh + +# TODO: copy config overrides from ENV vars + +# TODO: run celery in detached state +export SERVER_THREADS_AMOUNT=8 +# start up the web server + +/usr/bin/run-server.sh \ No newline at end of file diff --git a/common/superset/docker/docker-frontend.sh b/common/superset/docker/docker-frontend.sh new file mode 100644 index 0000000..6f96a1c --- /dev/null +++ b/common/superset/docker/docker-frontend.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +# Packages needed for puppeteer: +apt update +if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then + apt install -y chromium +fi + +if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then + cd /app/superset-frontend + npm install -f --no-optional --global webpack webpack-cli + npm install -f --no-optional + + echo "Running frontend" + npm run dev +else + echo "Skipping frontend build steps - YOU RUN IT MANUALLY ON THE HOST!" +fi \ No newline at end of file diff --git a/common/superset/docker/docker-init.sh b/common/superset/docker/docker-init.sh new file mode 100644 index 0000000..2029f47 --- /dev/null +++ b/common/superset/docker/docker-init.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +# +# Always install local overrides first +# +/app/docker/docker-bootstrap.sh + +STEP_CNT=4 + +echo_step() { +cat <