Merge pull request #109 from Project-OMOTES/108-simplify-postgres-inf…

…luxdb-and-rabbitmq-setup-script-to-also-correspond-with-httpsgithubcomproject-omotesomotes-helm

use setup scripts directly for postgres influx rabbitmq

docker-compose.override.setup.yml

@@ -0,0 +1,25 @@
+services:
+
+  rabbitmq:
+    volumes:
+      - "./rabbitmq/rabbitmq-init.sh:/setup/rabbitmq-init.sh"
+    environment:
+      RABBITMQ_OMOTES_USER_NAME: ${RABBITMQ_OMOTES_USER_NAME}
+      RABBITMQ_OMOTES_USER_PASSWORD: ${RABBITMQ_OMOTES_USER_PASSWORD}
+      RABBITMQ_CELERY_USER_NAME: ${RABBITMQ_CELERY_USER_NAME}
+      RABBITMQ_CELERY_USER_PASSWORD: ${RABBITMQ_CELERY_USER_PASSWORD}
+
+  omotes_influxdb:
+    volumes:
+      - "./influxdb/influxdb-init.sh:/setup/influxdb-init.sh"
+    environment:
+      INFLUXDB_WRITE_USER: ${INFLUXDB_WRITE_USER}
+      INFLUXDB_WRITE_USER_PASSWORD: ${INFLUXDB_WRITE_USER_PASSWORD}
+      INFLUXDB_FRONTEND_ADMIN_USER: ${INFLUXDB_FRONTEND_ADMIN_USER}
+      INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD}
+      INFLUXDB_PORT: ${INFLUXDB_PORT}
+
+  orchestrator_postgres_db:
+    volumes:
+      - "./postgres/postgres-init.sql:/setup/init.sql"
+

influxdb/influxdb-init.sh

@@ -0,0 +1,9 @@
+INFLUXDB_EXEC="influx -username $INFLUXDB_ADMIN_USER -password $INFLUXDB_ADMIN_PASSWORD -host omotes_influxdb -port $INFLUXDB_PORT -execute"
+$INFLUXDB_EXEC "CREATE USER $INFLUXDB_WRITE_USER WITH PASSWORD '$INFLUXDB_WRITE_USER_PASSWORD'";
+$INFLUXDB_EXEC "GRANT WRITE ON omotes_timeseries TO $INFLUXDB_WRITE_USER";
+$INFLUXDB_EXEC "SET PASSWORD FOR $INFLUXDB_WRITE_USER = '$INFLUXDB_WRITE_USER_PASSWORD'";
+echo "Influxdb user '$INFLUXDB_WRITE_USER' created/updated.";
+
+$INFLUXDB_EXEC "CREATE USER $INFLUXDB_FRONTEND_ADMIN_USER WITH PASSWORD '$INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD' WITH ALL PRIVILEGES";
+$INFLUXDB_EXEC "SET PASSWORD FOR $INFLUXDB_FRONTEND_ADMIN_USER = '$INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD'";
+echo "Influxdb user '$INFLUXDB_FRONTEND_ADMIN_USER' created/updated.";

postgres/postgres-init.sql

@@ -0,0 +1,8 @@
+CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
+ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
+GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME;
+GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME;

rabbitmq/rabbitmq-init.sh

@@ -0,0 +1,19 @@
+rabbitmqctl add_vhost omotes;
+rabbitmqctl set_permissions --vhost omotes root ".*" ".*" ".*";
+rabbitmqctl add_vhost omotes_celery;
+rabbitmqctl set_permissions --vhost omotes_celery root ".*" ".*" ".*";
+rabbitmqctl add_user --vhost omotes "$RABBITMQ_OMOTES_USER_NAME" "$RABBITMQ_OMOTES_USER_PASSWORD";
+rabbitmqctl set_permissions --vhost omotes "$RABBITMQ_OMOTES_USER_NAME" ".*" ".*" ".*";
+rabbitmqctl add_user --vhost omotes_celery "$RABBITMQ_CELERY_USER_NAME" "$RABBITMQ_CELERY_USER_PASSWORD";
+rabbitmqctl set_permissions --vhost omotes_celery "$RABBITMQ_CELERY_USER_NAME" ".*" ".*" ".*";
+
+#_________________________________________________________________________________________________
+# QUEUE MIGRATIONS
+
+# mvp.4.RC2
+# Ticket 84 84-extend-available-workflow-functionality-to-support-multiple-sdks-at-once
+
+# Remove available_workflows & request_available_workflows durable queues if they exist.
+# Messages may be dropped without repercussions.
+rabbitmqctl delete_queue --vhost omotes available_workflows
+rabbitmqctl delete_queue --vhost omotes request_available_workflows

scripts/setup.sh

@@ -1,8 +1,34 @@
 #!/bin/bash
 
-ENV_FILE="./.env"
-DOCKER_COMPOSE_FILE="./docker-compose.yml"
+# optionally set env file and docker compose file from arguments
+ENV_FILE=${1:-"./.env"}
+DOCKER_COMPOSE_FILE=${2:-"./docker-compose.yml -f ./docker-compose.override.setup.yml"}
 
-./scripts/setup_orchestrator_postgres_db.sh $ENV_FILE $DOCKER_COMPOSE_FILE
-./scripts/setup_rabbitmq.sh $ENV_FILE $DOCKER_COMPOSE_FILE
-./scripts/setup_influxdb.sh $ENV_FILE $DOCKER_COMPOSE_FILE
+. "$(dirname "$0")"/_select_docker_compose.sh
+. "$(dirname "$0")"/_load_dot_env.sh $ENV_FILE
+
+echo "Using docker compose file at: $DOCKER_COMPOSE_FILE"
+
+# stop system before setup
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down
+
+# add postgres user with privileges
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait orchestrator_postgres_db
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec orchestrator_postgres_db psql \
+  -d omotes_jobs \
+  -v PG_USERNAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \
+  -v PG_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \
+  -v PG_DB=omotes_jobs \
+  -f /setup/init.sql
+
+# add rabbitmq 'omotes' and 'celery' vhosts and users
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE up --wait rabbitmq
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec rabbitmq /bin/sh -c "./setup/rabbitmq-init.sh"
+
+# add influxdb users with write access for optimizer/simulator and with admin rights
+# for the frontend (root admin user via env vars)
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE up --wait omotes_influxdb
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_influxdb /bin/sh -c "./setup/influxdb-init.sh"
+
+# stop system after setup
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE down

scripts/test_system.sh

@@ -10,7 +10,5 @@ cp .env.template ${ENV_FILE}
 sed -i 's/LOG_LEVEL=[a-z]*/LOG_LEVEL=WARNING/gi' ${ENV_FILE}
 
 $DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE  down -v
-./scripts/setup_orchestrator_postgres_db.sh $ENV_FILE $DOCKER_COMPOSE_FILE
-./scripts/setup_rabbitmq.sh $ENV_FILE $DOCKER_COMPOSE_FILE
-./scripts/setup_influxdb.sh $ENV_FILE $DOCKER_COMPOSE_FILE
+./scripts/setup.sh $ENV_FILE "./docker-compose.yml -f ./docker-compose.override.setup.yml"
 $DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE up --build --abort-on-container-exit

scripts/test_system_dev.sh

@@ -10,7 +10,5 @@ cp .env-template ${ENV_FILE}
 sed -i 's/LOG_LEVEL=[a-z]*/LOG_LEVEL=WARNING/gi' ${ENV_FILE}
 
 $DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE  down -v
-./scripts/setup_orchestrator_postgres_db.sh $ENV_FILE $DOCKER_COMPOSE_FILE
-./scripts/setup_rabbitmq.sh $ENV_FILE $DOCKER_COMPOSE_FILE
-./scripts/setup_influxdb.sh $ENV_FILE $DOCKER_COMPOSE_FILE
+./scripts/setup.sh $ENV_FILE "./docker-compose.yml -f ./docker-compose.override.setup.yml"
 $DOCKER_COMPOSE --env-file ${ENV_FILE} -f $DOCKER_COMPOSE_FILE up --build --abort-on-container-exit