Skip to content

Commit

Permalink
(#34) | upgrade steps as per ecr repo
Browse files Browse the repository at this point in the history
  • Loading branch information
@vedfordev
vedfordev committed Nov 15, 2024
1 parent 09b29f7 commit a5c6ce9
Show file tree
Hide file tree
Showing 3 changed files with 143 additions and 88 deletions.
24 changes: 21 additions & 3 deletions reportingSystem/superset/Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
IMAGE := public.ecr.aws/m1p7h9p1/avniproject/reporting-superset
IMAGE := avniproject/reporting-superset
TAG ?= 4.0.1

build-image:
Expand All @@ -18,8 +18,26 @@ delete-image:
fi

push-image:
docker push $(IMAGE):$(TAG)

@if [ -z "$(REPO_URI)" ]; then \
echo "REPO_URI is not set"; \
exit 1; \
fi
@if docker image inspect $(REPO_URI)/$(IMAGE):$(TAG) > /dev/null 2>&1; then \
echo "Image $(REPO_URI)/$(IMAGE):$(TAG) found. deleting from the local"; \
docker image rm -f $(IMAGE):$(TAG); \
else \
echo "Image $(REPO_URI)/$(IMAGE):$(TAG) not found in local."; \
fi
docker tag $(IMAGE):$(TAG) $(REPO_URI)/$(IMAGE):$(TAG)
echo "pushing image $(REPO_URI)/$(IMAGE):$(TAG)";
docker push $(REPO_URI)/$(IMAGE):$(TAG)

delete-repo-image:
@if [ -z "$(REPO_URI)" ]; then \
echo "REPO_URI is not set"; \
exit 1; \
fi
docker image rm $(REPO_URI)/$(IMAGE):$(TAG)

inspect-image:
docker image inspect $(IMAGE):$(TAG)
Expand Down
147 changes: 84 additions & 63 deletions reportingSystem/superset/assets/reset_sequence.sql
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
SELECT * FROM information_schema.sequences;
SELECT * FROM information_schema.sequences order by sequence_name;

select table_name from information_schema.tables where table_schema = 'public' order by 1;

select *
from alert_owner_id_seq;
select 'SELECT SETVAL(''' || table_name || '_id_seq'' ,(SELECT GREATEST(MAX(id), nextval(''' || table_name || '_id_seq'')-1) from ' || table_name || '));'
from information_schema.tables
where table_schema = 'public'
and table_name in(
SELECT split_part(sequence_name,'_id_seq',1) FROM information_schema.sequences
)
order by 1;
-- to generate stamtement


SELECT
Expand All @@ -23,63 +28,79 @@ ORDER BY



select SETVAL('ab_permission_id_seq', (SELECT MAX(id) FROM ab_permission));
select SETVAL('ab_permission_view_id_seq', (SELECT MAX(id) FROM ab_permission_view));
select SETVAL('ab_permission_view_role_id_seq', (SELECT MAX(id) FROM ab_permission_view_role));
select SETVAL('ab_register_user_id_seq', (SELECT MAX(id) FROM ab_register_user));
select SETVAL('ab_role_id_seq', (SELECT MAX(id) FROM ab_role));
select SETVAL('ab_user_id_seq', (SELECT MAX(id) FROM ab_user));
select SETVAL('ab_user_role_id_seq', (SELECT MAX(id) FROM ab_user_role));
select SETVAL('ab_view_menu_id_seq', (SELECT MAX(id) FROM ab_view_menu));
select SETVAL('access_request_id_seq', (SELECT MAX(id) FROM access_request));
select SETVAL('alert_logs_id_seq', (SELECT MAX(id) FROM alert_logs));
select SETVAL('alert_owner_id_seq', (SELECT MAX(id) FROM alert_owner));
select SETVAL('alerts_id_seq', (SELECT MAX(id) FROM alerts));
select SETVAL('annotation_id_seq', (SELECT MAX(id) FROM annotation));
select SETVAL('annotation_layer_id_seq', (SELECT MAX(id) FROM annotation_layer));
select SETVAL('cache_keys_id_seq', (SELECT MAX(id) FROM cache_keys));
select SETVAL('clusters_id_seq', (SELECT MAX(id) FROM clusters));
select SETVAL('columns_id_seq', (SELECT MAX(id) FROM columns));
select SETVAL('css_templates_id_seq', (SELECT MAX(id) FROM css_templates));
select SETVAL('dashboard_email_schedules_id_seq', (SELECT MAX(id) FROM dashboard_email_schedules));
select SETVAL('dashboard_roles_id_seq', (SELECT MAX(id) FROM dashboard_roles));
select SETVAL('dashboard_slices_id_seq', (SELECT MAX(id) FROM dashboard_slices));
select SETVAL('dashboard_user_id_seq', (SELECT MAX(id) FROM dashboard_user));
select SETVAL('dashboards_id_seq', (SELECT MAX(id) FROM dashboards));
select SETVAL('datasources_id_seq', (SELECT MAX(id) FROM datasources));
select SETVAL('dbs_id_seq', (SELECT MAX(id) FROM dbs));
select SETVAL('druiddatasource_user_id_seq', (SELECT MAX(id) FROM druiddatasource_user));
select SETVAL('dynamic_plugin_id_seq', (SELECT MAX(id) FROM dynamic_plugin));
select SETVAL('favstar_id_seq', (SELECT MAX(id) FROM favstar));
select SETVAL('filter_sets_id_seq', (SELECT MAX(id) FROM filter_sets));
select SETVAL('key_value_id_seq', (SELECT MAX(id) FROM key_value));
select SETVAL('keyvalue_id_seq', (SELECT MAX(id) FROM keyvalue));
select SETVAL('logs_id_seq', (SELECT MAX(id) FROM logs));
select SETVAL('metrics_id_seq', (SELECT MAX(id) FROM metrics));
select SETVAL('query_id_seq', (SELECT MAX(id) FROM query));
select SETVAL('report_execution_log_id_seq', (SELECT MAX(id) FROM report_execution_log));
select SETVAL('report_recipient_id_seq', (SELECT MAX(id) FROM report_recipient));
select SETVAL('report_schedule_id_seq', (SELECT MAX(id) FROM report_schedule));
select SETVAL('report_schedule_user_id_seq', (SELECT MAX(id) FROM report_schedule_user));
select SETVAL('rls_filter_roles_id_seq', (SELECT MAX(id) FROM rls_filter_roles));
select SETVAL('rls_filter_tables_id_seq', (SELECT MAX(id) FROM rls_filter_tables));
select SETVAL('row_level_security_filters_id_seq', (SELECT MAX(id) FROM row_level_security_filters));
select SETVAL('saved_query_id_seq', (SELECT MAX(id) FROM saved_query));
select SETVAL('sl_columns_id_seq', (SELECT MAX(id) FROM sl_columns));
select SETVAL('sl_datasets_id_seq', (SELECT MAX(id) FROM sl_datasets));
select SETVAL('sl_tables_id_seq', (SELECT MAX(id) FROM sl_tables));
select SETVAL('slice_email_schedules_id_seq', (SELECT MAX(id) FROM slice_email_schedules));
select SETVAL('slice_user_id_seq', (SELECT MAX(id) FROM slice_user));
select SETVAL('slices_id_seq', (SELECT MAX(id) FROM slices));
select SETVAL('sql_metrics_id_seq', (SELECT MAX(id) FROM sql_metrics));
select SETVAL('sql_observations_id_seq', (SELECT MAX(id) FROM sql_observations));
select SETVAL('sqlatable_user_id_seq', (SELECT MAX(id) FROM sqlatable_user));
select SETVAL('tab_state_id_seq', (SELECT MAX(id) FROM tab_state));
select SETVAL('table_columns_id_seq', (SELECT MAX(id) FROM table_columns));
select SETVAL('table_schema_id_seq', (SELECT MAX(id) FROM table_schema));
select SETVAL('tables_id_seq', (SELECT MAX(id) FROM tables));
select SETVAL('tag_id_seq', (SELECT MAX(id) FROM tag));
select SETVAL('tagged_object_id_seq', (SELECT MAX(id) FROM tagged_object));
select SETVAL('url_id_seq', (SELECT MAX(id) FROM url));
select SETVAL('user_attribute_id_seq', (SELECT MAX(id) FROM user_attribute));
SELECT SETVAL('ab_permission_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_permission_id_seq')-1) from ab_permission));
SELECT SETVAL('ab_permission_view_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_permission_view_id_seq')-1) from ab_permission_view));
SELECT SETVAL('ab_permission_view_role_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_permission_view_role_id_seq')-1) from ab_permission_view_role));
SELECT SETVAL('ab_register_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_register_user_id_seq')-1) from ab_register_user));
SELECT SETVAL('ab_role_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_role_id_seq')-1) from ab_role));
SELECT SETVAL('ab_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_user_id_seq')-1) from ab_user));
SELECT SETVAL('ab_user_role_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_user_role_id_seq')-1) from ab_user_role));
SELECT SETVAL('ab_view_menu_id_seq' ,(SELECT GREATEST(MAX(id), nextval('ab_view_menu_id_seq')-1) from ab_view_menu));
SELECT SETVAL('access_request_id_seq' ,(SELECT GREATEST(MAX(id), nextval('access_request_id_seq')-1) from access_request));
SELECT SETVAL('alert_logs_id_seq' ,(SELECT GREATEST(MAX(id), nextval('alert_logs_id_seq')-1) from alert_logs));
SELECT SETVAL('alert_owner_id_seq' ,(SELECT GREATEST(MAX(id), nextval('alert_owner_id_seq')-1) from alert_owner));
SELECT SETVAL('alerts_id_seq' ,(SELECT GREATEST(MAX(id), nextval('alerts_id_seq')-1) from alerts));
SELECT SETVAL('annotation_id_seq' ,(SELECT GREATEST(MAX(id), nextval('annotation_id_seq')-1) from annotation));
SELECT SETVAL('annotation_layer_id_seq' ,(SELECT GREATEST(MAX(id), nextval('annotation_layer_id_seq')-1) from annotation_layer));
SELECT SETVAL('cache_keys_id_seq' ,(SELECT GREATEST(MAX(id), nextval('cache_keys_id_seq')-1) from cache_keys));
SELECT SETVAL('clusters_id_seq' ,(SELECT GREATEST(MAX(id), nextval('clusters_id_seq')-1) from clusters));
SELECT SETVAL('columns_id_seq' ,(SELECT GREATEST(MAX(id), nextval('columns_id_seq')-1) from columns));
SELECT SETVAL('css_templates_id_seq' ,(SELECT GREATEST(MAX(id), nextval('css_templates_id_seq')-1) from css_templates));
SELECT SETVAL('dashboard_email_schedules_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_email_schedules_id_seq')-1) from dashboard_email_schedules));
SELECT SETVAL('dashboard_roles_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_roles_id_seq')-1) from dashboard_roles));
SELECT SETVAL('dashboard_slices_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_slices_id_seq')-1) from dashboard_slices));
SELECT SETVAL('dashboard_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboard_user_id_seq')-1) from dashboard_user));
SELECT SETVAL('dashboards_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dashboards_id_seq')-1) from dashboards));
SELECT SETVAL('datasources_id_seq' ,(SELECT GREATEST(MAX(id), nextval('datasources_id_seq')-1) from datasources));
SELECT SETVAL('dbs_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dbs_id_seq')-1) from dbs));
SELECT SETVAL('druiddatasource_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('druiddatasource_user_id_seq')-1) from druiddatasource_user));
SELECT SETVAL('dynamic_plugin_id_seq' ,(SELECT GREATEST(MAX(id), nextval('dynamic_plugin_id_seq')-1) from dynamic_plugin));
SELECT SETVAL('favstar_id_seq' ,(SELECT GREATEST(MAX(id), nextval('favstar_id_seq')-1) from favstar));
SELECT SETVAL('filter_sets_id_seq' ,(SELECT GREATEST(MAX(id), nextval('filter_sets_id_seq')-1) from filter_sets));
SELECT SETVAL('key_value_id_seq' ,(SELECT GREATEST(MAX(id), nextval('key_value_id_seq')-1) from key_value));
SELECT SETVAL('keyvalue_id_seq' ,(SELECT GREATEST(MAX(id), nextval('keyvalue_id_seq')-1) from keyvalue));
SELECT SETVAL('logs_id_seq' ,(SELECT GREATEST(MAX(id), nextval('logs_id_seq')-1) from logs));
SELECT SETVAL('metrics_id_seq' ,(SELECT GREATEST(MAX(id), nextval('metrics_id_seq')-1) from metrics));
SELECT SETVAL('query_id_seq' ,(SELECT GREATEST(MAX(id), nextval('query_id_seq')-1) from query));
SELECT SETVAL('report_execution_log_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_execution_log_id_seq')-1) from report_execution_log));
SELECT SETVAL('report_recipient_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_recipient_id_seq')-1) from report_recipient));
SELECT SETVAL('report_schedule_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_schedule_id_seq')-1) from report_schedule));
SELECT SETVAL('report_schedule_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('report_schedule_user_id_seq')-1) from report_schedule_user));
SELECT SETVAL('rls_filter_roles_id_seq' ,(SELECT GREATEST(MAX(id), nextval('rls_filter_roles_id_seq')-1) from rls_filter_roles));
SELECT SETVAL('rls_filter_tables_id_seq' ,(SELECT GREATEST(MAX(id), nextval('rls_filter_tables_id_seq')-1) from rls_filter_tables));
SELECT SETVAL('row_level_security_filters_id_seq' ,(SELECT GREATEST(MAX(id), nextval('row_level_security_filters_id_seq')-1) from row_level_security_filters));
SELECT SETVAL('saved_query_id_seq' ,(SELECT GREATEST(MAX(id), nextval('saved_query_id_seq')-1) from saved_query));
SELECT SETVAL('sl_columns_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sl_columns_id_seq')-1) from sl_columns));
SELECT SETVAL('sl_datasets_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sl_datasets_id_seq')-1) from sl_datasets));
SELECT SETVAL('sl_tables_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sl_tables_id_seq')-1) from sl_tables));
SELECT SETVAL('slice_email_schedules_id_seq' ,(SELECT GREATEST(MAX(id), nextval('slice_email_schedules_id_seq')-1) from slice_email_schedules));
SELECT SETVAL('slice_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('slice_user_id_seq')-1) from slice_user));
SELECT SETVAL('slices_id_seq' ,(SELECT GREATEST(MAX(id), nextval('slices_id_seq')-1) from slices));
SELECT SETVAL('sql_metrics_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sql_metrics_id_seq')-1) from sql_metrics));
SELECT SETVAL('sql_observations_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sql_observations_id_seq')-1) from sql_observations));
SELECT SETVAL('sqlatable_user_id_seq' ,(SELECT GREATEST(MAX(id), nextval('sqlatable_user_id_seq')-1) from sqlatable_user));
SELECT SETVAL('tab_state_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tab_state_id_seq')-1) from tab_state));
SELECT SETVAL('table_columns_id_seq' ,(SELECT GREATEST(MAX(id), nextval('table_columns_id_seq')-1) from table_columns));
SELECT SETVAL('table_schema_id_seq' ,(SELECT GREATEST(MAX(id), nextval('table_schema_id_seq')-1) from table_schema));
SELECT SETVAL('tables_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tables_id_seq')-1) from tables));
SELECT SETVAL('tag_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tag_id_seq')-1) from tag));
SELECT SETVAL('tagged_object_id_seq' ,(SELECT GREATEST(MAX(id), nextval('tagged_object_id_seq')-1) from tagged_object));
SELECT SETVAL('url_id_seq' ,(SELECT GREATEST(MAX(id), nextval('url_id_seq')-1) from url));
SELECT SETVAL('user_attribute_id_seq' ,(SELECT GREATEST(MAX(id), nextval('user_attribute_id_seq')-1) from user_attribute));


SELECT
sequence_schema,
sequence_name,
last_value AS current_value
FROM
information_schema.sequences
JOIN pg_sequences ON sequence_schema = schemaname AND sequence_name = sequencename
JOIN pg_class ON pg_class.relname = sequence_name
JOIN pg_sequence ON pg_sequence.seqrelid = pg_class.oid
where last_value is null
ORDER BY
sequence_schema,
sequence_name
;
-- it should shows 0 result at end if not. no move further and check the sequence.
60 changes: 38 additions & 22 deletions reportingSystem/superset_db_sqllite_to_postgres.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ superset db upgrade

4. change source & destination in [/assets/pgloaderdataonly.conf](https://github.com/avniproject/avni-infra/blob/master/reportingSystem/superset/assets/pgloaderdataonly.conf) tunnel the postgres db and transfer the data.
```shell
## change as per environment
ssh -o ConnectTimeout=30 -N -L 5433:prereleasedb.avniproject.org:5432 [email protected] -i ~/.ssh/openchs-infra.pem
## go to second windoow
pgloader pgloaderdataonly.conf | tee db_transfer.txt
Expand All @@ -78,49 +79,64 @@ superset db init
```

---
### Upgrate to superset docker 4.0.1
### Upgrate to superset docker 4.0.1 (from avni ecr image)


1. stop the superset_2.0.1. copy data from [/assets/superset_config.py](https://github.com/avniproject/avni-infra/blob/master/reportingSystem/superset/assets/superset_config.py) and add require flags and configuration
1. stop the superset_2.0.1.
```shell
docker stop superset_2.0.1
#check /home/ubuntu/supersetdata exist or not if not then create
cd /home/ubuntu/supersetdata
vi superset_config.py
```

2. get logo of avni
2. image push to ecr repo (do if image is not set)
```shell
cd /home/ubuntu/supersetdata
wget https://avniproject.org/static/avni-logo-color-b42a730b01c55efe37722027d9cddd95.png
mv avni-logo-color-b42a730b01c55efe37722027d9cddd95.png avni.png
wget https://app.avniproject.org/favicon.ico
mv favicon.ico avni-favicon.ico
aws configure #if not done
aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin 118388513628.dkr.ecr.ap-south-1.amazonaws.com
make push-image REPO_URI=118388513628.dkr.ecr.ap-south-1.amazonaws.com
```

3. run docker for 4.0.1
3. copy environment file to supersetdata and configure aws [AWS CLI download reference](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
```shell
aws --version
# if error then install aws cli
aws configure
#provide access key and secret
mkdir supersetdata
cd supersetdata
# paste file superset.env from secrets
```

4. run docker for 4.0.1
```shell
sudo mkdir -p /var/log/superset

sudo chmod -R 777 /var/log/superset

aws ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin 118388513628.dkr.ecr.ap-south-1.amazonaws.com

docker pull 118388513628.dkr.ecr.ap-south-1.amazonaws.com/avniproject/reporting-superset:4.0.1

docker run -d -p 8088:8088 \
--network superset_network \
--name superset_4.0.1 \
-e SUPERSET_CONFIG_PATH=/app/superset_config.py \
-v /home/ubuntu/supersetdata/superset_config.py:/app/superset_config.py \
-v /home/ubuntu/supersetdata/avni.png:/app/superset/static/assets/images/avni.png \
-v /home/ubuntu/supersetdata/avni-favicon.ico:/app/superset/static/assets/images/avni-favicon.ico \
apache/superset:4.0.1
docker logs -f superset_4.0.1
--name avnisuperset_4.0.1 \
--env-file ~/supersetdata/superset.env \
-v /var/log/superset:/app/superset_home \
118388513628.dkr.ecr.ap-south-1.amazonaws.com/avniproject/reporting-superset:4.0.1
docker logs -f superset_4.0.1

tail -f /var/log/superset/superset.log
```

4. important commands for docker container
5. important commands for docker container
```shell
docker logs -f superset_4.0.1
docker exec -it -u root superset_4.0.1 bash
docker start superset_4.0.1
docker stop superset_4.0.1
docker restart superset_4.0.1
docker inspect avnisuperset_4.0.1
```

5. generate certificate for testing superset (optional)
6. generate certificate for testing superset (optional)
```shell
sudo certbot --nginx -d test-reporting-superset.avniproject.org
sudo certbot renew --cert-name test-reporting-superset.avniproject.org
Expand Down

0 comments on commit a5c6ce9

Please sign in to comment.