From a5c6ce928a2a2c704e57cbd61a61557aa1f6faa1 Mon Sep 17 00:00:00 2001 From: vedfordev Date: Fri, 15 Nov 2024 15:39:21 +0530 Subject: [PATCH] (#34) | upgrade steps as per ecr repo --- reportingSystem/superset/Makefile | 24 ++- .../superset/assets/reset_sequence.sql | 147 ++++++++++-------- .../superset_db_sqllite_to_postgres.md | 60 ++++--- 3 files changed, 143 insertions(+), 88 deletions(-) diff --git a/reportingSystem/superset/Makefile b/reportingSystem/superset/Makefile index 3c267dc..9d5d923 100644 --- a/reportingSystem/superset/Makefile +++ b/reportingSystem/superset/Makefile @@ -1,4 +1,4 @@ -IMAGE := public.ecr.aws/m1p7h9p1/avniproject/reporting-superset +IMAGE := avniproject/reporting-superset TAG ?= 4.0.1 build-image: @@ -18,8 +18,26 @@ delete-image: fi push-image: - docker push $(IMAGE):$(TAG) - + @if [ -z "$(REPO_URI)" ]; then \ + echo "REPO_URI is not set"; \ + exit 1; \ + fi + @if docker image inspect $(REPO_URI)/$(IMAGE):$(TAG) > /dev/null 2>&1; then \ + echo "Image $(REPO_URI)/$(IMAGE):$(TAG) found. deleting from the local"; \ + docker image rm -f $(IMAGE):$(TAG); \ + else \ + echo "Image $(REPO_URI)/$(IMAGE):$(TAG) not found in local."; \ + fi + docker tag $(IMAGE):$(TAG) $(REPO_URI)/$(IMAGE):$(TAG) + echo "pushing image $(REPO_URI)/$(IMAGE):$(TAG)"; + docker push $(REPO_URI)/$(IMAGE):$(TAG) + +delete-repo-image: + @if [ -z "$(REPO_URI)" ]; then \ + echo "REPO_URI is not set"; \ + exit 1; \ + fi + docker image rm $(REPO_URI)/$(IMAGE):$(TAG) inspect-image: docker image inspect $(IMAGE):$(TAG) diff --git a/reportingSystem/superset/assets/reset_sequence.sql b/reportingSystem/superset/assets/reset_sequence.sql index 48ea232..5e3fa51 100644 --- a/reportingSystem/superset/assets/reset_sequence.sql +++ b/reportingSystem/superset/assets/reset_sequence.sql @@ -1,9 +1,14 @@ -SELECT * FROM information_schema.sequences; +SELECT * FROM information_schema.sequences order by sequence_name; -select table_name from information_schema.tables where table_schema = 'public' order by 1; -select * -from alert_owner_id_seq; +select 'SELECT SETVAL(''' || table_name || '_id_seq'' ,(SELECT GREATEST(MAX(id), nextval(''' || table_name || '_id_seq'')-1) from ' || table_name || '));' +from information_schema.tables +where table_schema = 'public' + and table_name in( + SELECT split_part(sequence_name,'_id_seq',1) FROM information_schema.sequences +) +order by 1; +-- to generate stamtement SELECT @@ -23,63 +28,79 @@ ORDER BY -select SETVAL('ab_permission_id_seq', (SELECT MAX(id) FROM ab_permission)); -select SETVAL('ab_permission_view_id_seq', (SELECT MAX(id) FROM ab_permission_view)); -select SETVAL('ab_permission_view_role_id_seq', (SELECT MAX(id) FROM ab_permission_view_role)); -select SETVAL('ab_register_user_id_seq', (SELECT MAX(id) FROM ab_register_user)); -select SETVAL('ab_role_id_seq', (SELECT MAX(id) FROM ab_role)); -select SETVAL('ab_user_id_seq', (SELECT MAX(id) FROM ab_user)); -select SETVAL('ab_user_role_id_seq', (SELECT MAX(id) FROM ab_user_role)); -select SETVAL('ab_view_menu_id_seq', (SELECT MAX(id) FROM ab_view_menu)); -select SETVAL('access_request_id_seq', (SELECT MAX(id) FROM access_request)); -select SETVAL('alert_logs_id_seq', (SELECT MAX(id) FROM alert_logs)); -select SETVAL('alert_owner_id_seq', (SELECT MAX(id) FROM alert_owner)); -select SETVAL('alerts_id_seq', (SELECT MAX(id) FROM alerts)); -select SETVAL('annotation_id_seq', (SELECT MAX(id) FROM annotation)); -select SETVAL('annotation_layer_id_seq', (SELECT MAX(id) FROM annotation_layer)); -select SETVAL('cache_keys_id_seq', (SELECT MAX(id) FROM cache_keys)); -select SETVAL('clusters_id_seq', (SELECT MAX(id) FROM clusters)); -select SETVAL('columns_id_seq', (SELECT MAX(id) FROM columns)); -select SETVAL('css_templates_id_seq', (SELECT MAX(id) FROM css_templates)); -select SETVAL('dashboard_email_schedules_id_seq', (SELECT MAX(id) FROM dashboard_email_schedules)); -select SETVAL('dashboard_roles_id_seq', (SELECT MAX(id) FROM dashboard_roles)); -select SETVAL('dashboard_slices_id_seq', (SELECT MAX(id) FROM dashboard_slices)); -select SETVAL('dashboard_user_id_seq', (SELECT MAX(id) FROM dashboard_user)); -select SETVAL('dashboards_id_seq', (SELECT MAX(id) FROM dashboards)); -select SETVAL('datasources_id_seq', (SELECT MAX(id) FROM datasources)); -select SETVAL('dbs_id_seq', (SELECT MAX(id) FROM dbs)); -select SETVAL('druiddatasource_user_id_seq', (SELECT MAX(id) FROM druiddatasource_user)); -select SETVAL('dynamic_plugin_id_seq', (SELECT MAX(id) FROM dynamic_plugin)); -select SETVAL('favstar_id_seq', (SELECT MAX(id) FROM favstar)); -select SETVAL('filter_sets_id_seq', (SELECT MAX(id) FROM filter_sets)); -select SETVAL('key_value_id_seq', (SELECT MAX(id) FROM key_value)); -select SETVAL('keyvalue_id_seq', (SELECT MAX(id) FROM keyvalue)); -select SETVAL('logs_id_seq', (SELECT MAX(id) FROM logs)); -select SETVAL('metrics_id_seq', (SELECT MAX(id) FROM metrics)); -select SETVAL('query_id_seq', (SELECT MAX(id) FROM query)); -select SETVAL('report_execution_log_id_seq', (SELECT MAX(id) FROM report_execution_log)); -select SETVAL('report_recipient_id_seq', (SELECT MAX(id) FROM report_recipient)); -select SETVAL('report_schedule_id_seq', (SELECT MAX(id) FROM report_schedule)); -select SETVAL('report_schedule_user_id_seq', (SELECT MAX(id) FROM report_schedule_user)); -select SETVAL('rls_filter_roles_id_seq', (SELECT MAX(id) FROM rls_filter_roles)); -select SETVAL('rls_filter_tables_id_seq', (SELECT MAX(id) FROM rls_filter_tables)); -select SETVAL('row_level_security_filters_id_seq', (SELECT MAX(id) FROM row_level_security_filters)); -select SETVAL('saved_query_id_seq', (SELECT MAX(id) FROM saved_query)); -select SETVAL('sl_columns_id_seq', (SELECT MAX(id) FROM sl_columns)); -select SETVAL('sl_datasets_id_seq', (SELECT MAX(id) FROM sl_datasets)); -select SETVAL('sl_tables_id_seq', (SELECT MAX(id) FROM sl_tables)); -select SETVAL('slice_email_schedules_id_seq', (SELECT MAX(id) FROM slice_email_schedules)); -select SETVAL('slice_user_id_seq', (SELECT MAX(id) FROM slice_user)); -select SETVAL('slices_id_seq', (SELECT MAX(id) FROM slices)); -select SETVAL('sql_metrics_id_seq', (SELECT MAX(id) FROM sql_metrics)); -select SETVAL('sql_observations_id_seq', (SELECT MAX(id) FROM sql_observations)); -select SETVAL('sqlatable_user_id_seq', (SELECT MAX(id) FROM sqlatable_user)); -select SETVAL('tab_state_id_seq', (SELECT MAX(id) FROM tab_state)); -select SETVAL('table_columns_id_seq', (SELECT MAX(id) FROM table_columns)); -select SETVAL('table_schema_id_seq', (SELECT MAX(id) FROM table_schema)); -select SETVAL('tables_id_seq', (SELECT MAX(id) FROM tables)); -select SETVAL('tag_id_seq', (SELECT MAX(id) FROM tag)); -select SETVAL('tagged_object_id_seq', (SELECT MAX(id) FROM tagged_object)); -select SETVAL('url_id_seq', (SELECT MAX(id) FROM url)); -select SETVAL('user_attribute_id_seq', (SELECT MAX(id) FROM user_attribute)); +SELECT SETVAL('ab_permission_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_permission_id_seq')-1) from ab_permission)); +SELECT SETVAL('ab_permission_view_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_permission_view_id_seq')-1) from ab_permission_view)); +SELECT SETVAL('ab_permission_view_role_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_permission_view_role_id_seq')-1) from ab_permission_view_role)); +SELECT SETVAL('ab_register_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_register_user_id_seq')-1) from ab_register_user)); +SELECT SETVAL('ab_role_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_role_id_seq')-1) from ab_role)); +SELECT SETVAL('ab_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_user_id_seq')-1) from ab_user)); +SELECT SETVAL('ab_user_role_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_user_role_id_seq')-1) from ab_user_role)); +SELECT SETVAL('ab_view_menu_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_view_menu_id_seq')-1) from ab_view_menu)); +SELECT SETVAL('access_request_id_seq' ,(SELECT GREATEST(MAX(id), nextval('access_request_id_seq')-1) from access_request)); +SELECT SETVAL('alert_logs_id_seq' ,(SELECT GREATEST(MAX(id), nextval('alert_logs_id_seq')-1) from alert_logs)); +SELECT SETVAL('alert_owner_id_seq' ,(SELECT GREATEST(MAX(id), nextval('alert_owner_id_seq')-1) from alert_owner)); +SELECT SETVAL('alerts_id_seq' ,(SELECT GREATEST(MAX(id), nextval('alerts_id_seq')-1) from alerts)); +SELECT SETVAL('annotation_id_seq' ,(SELECT GREATEST(MAX(id), nextval('annotation_id_seq')-1) from annotation)); +SELECT SETVAL('annotation_layer_id_seq' ,(SELECT GREATEST(MAX(id), nextval('annotation_layer_id_seq')-1) from annotation_layer)); +SELECT SETVAL('cache_keys_id_seq' ,(SELECT GREATEST(MAX(id), nextval('cache_keys_id_seq')-1) from cache_keys)); +SELECT SETVAL('clusters_id_seq' ,(SELECT GREATEST(MAX(id), nextval('clusters_id_seq')-1) from clusters)); +SELECT SETVAL('columns_id_seq' ,(SELECT GREATEST(MAX(id), nextval('columns_id_seq')-1) from columns)); +SELECT SETVAL('css_templates_id_seq' ,(SELECT GREATEST(MAX(id), nextval('css_templates_id_seq')-1) from css_templates)); +SELECT SETVAL('dashboard_email_schedules_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_email_schedules_id_seq')-1) from dashboard_email_schedules)); +SELECT SETVAL('dashboard_roles_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_roles_id_seq')-1) from dashboard_roles)); +SELECT SETVAL('dashboard_slices_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_slices_id_seq')-1) from dashboard_slices)); +SELECT SETVAL('dashboard_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_user_id_seq')-1) from dashboard_user)); +SELECT SETVAL('dashboards_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboards_id_seq')-1) from dashboards)); +SELECT SETVAL('datasources_id_seq' ,(SELECT GREATEST(MAX(id), nextval('datasources_id_seq')-1) from datasources)); +SELECT SETVAL('dbs_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dbs_id_seq')-1) from dbs)); +SELECT SETVAL('druiddatasource_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('druiddatasource_user_id_seq')-1) from druiddatasource_user)); +SELECT SETVAL('dynamic_plugin_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dynamic_plugin_id_seq')-1) from dynamic_plugin)); +SELECT SETVAL('favstar_id_seq' ,(SELECT GREATEST(MAX(id), nextval('favstar_id_seq')-1) from favstar)); +SELECT SETVAL('filter_sets_id_seq' ,(SELECT GREATEST(MAX(id), nextval('filter_sets_id_seq')-1) from filter_sets)); +SELECT SETVAL('key_value_id_seq' ,(SELECT GREATEST(MAX(id), nextval('key_value_id_seq')-1) from key_value)); +SELECT SETVAL('keyvalue_id_seq' ,(SELECT GREATEST(MAX(id), nextval('keyvalue_id_seq')-1) from keyvalue)); +SELECT SETVAL('logs_id_seq' ,(SELECT GREATEST(MAX(id), nextval('logs_id_seq')-1) from logs)); +SELECT SETVAL('metrics_id_seq' ,(SELECT GREATEST(MAX(id), nextval('metrics_id_seq')-1) from metrics)); +SELECT SETVAL('query_id_seq' ,(SELECT GREATEST(MAX(id), nextval('query_id_seq')-1) from query)); +SELECT SETVAL('report_execution_log_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_execution_log_id_seq')-1) from report_execution_log)); +SELECT SETVAL('report_recipient_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_recipient_id_seq')-1) from report_recipient)); +SELECT SETVAL('report_schedule_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_schedule_id_seq')-1) from report_schedule)); +SELECT SETVAL('report_schedule_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_schedule_user_id_seq')-1) from report_schedule_user)); +SELECT SETVAL('rls_filter_roles_id_seq' ,(SELECT GREATEST(MAX(id), nextval('rls_filter_roles_id_seq')-1) from rls_filter_roles)); +SELECT SETVAL('rls_filter_tables_id_seq' ,(SELECT GREATEST(MAX(id), nextval('rls_filter_tables_id_seq')-1) from rls_filter_tables)); +SELECT SETVAL('row_level_security_filters_id_seq' ,(SELECT GREATEST(MAX(id), nextval('row_level_security_filters_id_seq')-1) from row_level_security_filters)); +SELECT SETVAL('saved_query_id_seq' ,(SELECT GREATEST(MAX(id), nextval('saved_query_id_seq')-1) from saved_query)); +SELECT SETVAL('sl_columns_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sl_columns_id_seq')-1) from sl_columns)); +SELECT SETVAL('sl_datasets_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sl_datasets_id_seq')-1) from sl_datasets)); +SELECT SETVAL('sl_tables_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sl_tables_id_seq')-1) from sl_tables)); +SELECT SETVAL('slice_email_schedules_id_seq' ,(SELECT GREATEST(MAX(id), nextval('slice_email_schedules_id_seq')-1) from slice_email_schedules)); +SELECT SETVAL('slice_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('slice_user_id_seq')-1) from slice_user)); +SELECT SETVAL('slices_id_seq' ,(SELECT GREATEST(MAX(id), nextval('slices_id_seq')-1) from slices)); +SELECT SETVAL('sql_metrics_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sql_metrics_id_seq')-1) from sql_metrics)); +SELECT SETVAL('sql_observations_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sql_observations_id_seq')-1) from sql_observations)); +SELECT SETVAL('sqlatable_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sqlatable_user_id_seq')-1) from sqlatable_user)); +SELECT SETVAL('tab_state_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tab_state_id_seq')-1) from tab_state)); +SELECT SETVAL('table_columns_id_seq' ,(SELECT GREATEST(MAX(id), nextval('table_columns_id_seq')-1) from table_columns)); +SELECT SETVAL('table_schema_id_seq' ,(SELECT GREATEST(MAX(id), nextval('table_schema_id_seq')-1) from table_schema)); +SELECT SETVAL('tables_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tables_id_seq')-1) from tables)); +SELECT SETVAL('tag_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tag_id_seq')-1) from tag)); +SELECT SETVAL('tagged_object_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tagged_object_id_seq')-1) from tagged_object)); +SELECT SETVAL('url_id_seq' ,(SELECT GREATEST(MAX(id), nextval('url_id_seq')-1) from url)); +SELECT SETVAL('user_attribute_id_seq' ,(SELECT GREATEST(MAX(id), nextval('user_attribute_id_seq')-1) from user_attribute)); + +SELECT + sequence_schema, + sequence_name, + last_value AS current_value +FROM + information_schema.sequences + JOIN pg_sequences ON sequence_schema = schemaname AND sequence_name = sequencename + JOIN pg_class ON pg_class.relname = sequence_name + JOIN pg_sequence ON pg_sequence.seqrelid = pg_class.oid +where last_value is null +ORDER BY + sequence_schema, + sequence_name +; +-- it should shows 0 result at end if not. no move further and check the sequence. \ No newline at end of file diff --git a/reportingSystem/superset_db_sqllite_to_postgres.md b/reportingSystem/superset_db_sqllite_to_postgres.md index e29b83e..2bf8091 100644 --- a/reportingSystem/superset_db_sqllite_to_postgres.md +++ b/reportingSystem/superset_db_sqllite_to_postgres.md @@ -57,6 +57,7 @@ superset db upgrade 4. change source & destination in [/assets/pgloaderdataonly.conf](https://github.com/avniproject/avni-infra/blob/master/reportingSystem/superset/assets/pgloaderdataonly.conf) tunnel the postgres db and transfer the data. ```shell +## change as per environment ssh -o ConnectTimeout=30 -N -L 5433:prereleasedb.avniproject.org:5432 ubuntu@ssh.prerelease.avniproject.org -i ~/.ssh/openchs-infra.pem ## go to second windoow pgloader pgloaderdataonly.conf | tee db_transfer.txt @@ -78,49 +79,64 @@ superset db init ``` --- -### Upgrate to superset docker 4.0.1 +### Upgrate to superset docker 4.0.1 (from avni ecr image) -1. stop the superset_2.0.1. copy data from [/assets/superset_config.py](https://github.com/avniproject/avni-infra/blob/master/reportingSystem/superset/assets/superset_config.py) and add require flags and configuration +1. stop the superset_2.0.1. ```shell docker stop superset_2.0.1 -#check /home/ubuntu/supersetdata exist or not if not then create -cd /home/ubuntu/supersetdata -vi superset_config.py ``` -2. get logo of avni +2. image push to ecr repo (do if image is not set) ```shell -cd /home/ubuntu/supersetdata -wget https://avniproject.org/static/avni-logo-color-b42a730b01c55efe37722027d9cddd95.png -mv avni-logo-color-b42a730b01c55efe37722027d9cddd95.png avni.png -wget https://app.avniproject.org/favicon.ico -mv favicon.ico avni-favicon.ico +aws configure #if not done +aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin 118388513628.dkr.ecr.ap-south-1.amazonaws.com +make push-image REPO_URI=118388513628.dkr.ecr.ap-south-1.amazonaws.com ``` -3. run docker for 4.0.1 +3. copy environment file to supersetdata and configure aws [AWS CLI download reference](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) ```shell +aws --version +# if error then install aws cli +aws configure +#provide access key and secret +mkdir supersetdata +cd supersetdata +# paste file superset.env from secrets +``` + +4. run docker for 4.0.1 +```shell +sudo mkdir -p /var/log/superset + +sudo chmod -R 777 /var/log/superset + +aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin 118388513628.dkr.ecr.ap-south-1.amazonaws.com + +docker pull 118388513628.dkr.ecr.ap-south-1.amazonaws.com/avniproject/reporting-superset:4.0.1 + docker run -d -p 8088:8088 \ - --network superset_network \ - --name superset_4.0.1 \ - -e SUPERSET_CONFIG_PATH=/app/superset_config.py \ - -v /home/ubuntu/supersetdata/superset_config.py:/app/superset_config.py \ - -v /home/ubuntu/supersetdata/avni.png:/app/superset/static/assets/images/avni.png \ - -v /home/ubuntu/supersetdata/avni-favicon.ico:/app/superset/static/assets/images/avni-favicon.ico \ - apache/superset:4.0.1 -docker logs -f superset_4.0.1 + --name avnisuperset_4.0.1 \ + --env-file ~/supersetdata/superset.env \ + -v /var/log/superset:/app/superset_home \ + 118388513628.dkr.ecr.ap-south-1.amazonaws.com/avniproject/reporting-superset:4.0.1 + +docker logs -f superset_4.0.1 + +tail -f /var/log/superset/superset.log ``` -4. important commands for docker container +5. important commands for docker container ```shell docker logs -f superset_4.0.1 docker exec -it -u root superset_4.0.1 bash docker start superset_4.0.1 docker stop superset_4.0.1 docker restart superset_4.0.1 +docker inspect avnisuperset_4.0.1 ``` -5. generate certificate for testing superset (optional) +6. generate certificate for testing superset (optional) ```shell sudo certbot --nginx -d test-reporting-superset.avniproject.org sudo certbot renew --cert-name test-reporting-superset.avniproject.org