Synth data migrator

.circleci/config.yml

@@ -38,7 +38,7 @@ defaults:
   - &wait_for_flowdb
     name: Wait for flowdb to start
     command: |
-      dockerize -wait tcp://localhost:5432 -timeout 10m
+      dockerize -wait tcp://localhost:5432 -timeout 20m
   - &run_always_org_context
     context: org-global
     filters:
@@ -818,6 +818,7 @@ jobs:
       name: python_with_flowdb
       flowdb_image: "testdata"
       python_version: "3.8.5"
+      num_days: 7 # To avoid overriding fixed number of days
     working_directory: /home/circleci/project/integration_tests
     steps:
       - checkout:

flowdb/testdata/bin/9910_migrate_test_data.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+
+set -e
+export PGUSER="$POSTGRES_USER"
+
+#
+#  Migrate synthetic data.
+#
+
+pipenv run python /docker-entrypoint-initdb.d/migrate_synth_data.py

flowdb_synthetic_data.Dockerfile

@@ -38,6 +38,8 @@ RUN mkdir -p /docker-entrypoint-initdb.d/sql/syntheticdata/ && \
 
 COPY --chown=postgres flowdb/testdata/bin/9900_ingest_synthetic_data.sh /docker-entrypoint-initdb.d/
 COPY --chown=postgres flowdb/testdata/bin/9800_population_density.sql.gz /docker-entrypoint-initdb.d/
+COPY --chown=postgres flowdb/testdata/bin/9910_migrate_test_data.sh /docker-entrypoint-initdb.d/
+COPY --chown=postgres flowdb/testdata/bin/migrate_synth_data.py /docker-entrypoint-initdb.d/
 COPY --chown=postgres flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh /docker-entrypoint-initdb.d/
 COPY --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/
 

flowdb_synthetic_data.Dockerfile.dockerignore

@@ -12,6 +12,8 @@
 !flowdb/testdata/bin/9900_ingest_synthetic_data.sh
 !flowdb/testdata/bin/9800_population_density.sql.gz
 !flowdb/testdata/bin/generate_synthetic_data*.py
+!flowdb/testdata/bin/9910_migrate_test_data.sh
+!flowdb/testdata/bin/migrate_synth_data.py
 !flowdb/testdata/test_data/sql/admin*.sql
 !flowdb/testdata/synthetic_data/data/NPL_admbnda_adm3_Districts_simplified.geojson
 !flowdb/testdata/synthetic_data/Pipfile*

flowdb_testdata.Dockerfile.dockerignore

@@ -12,7 +12,8 @@
 #
 !flowdb/testdata/bin/9900_ingest_test_data.sh
 !flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh
-!flowdb/testdata/bin/9910_migrate_test_data.sql
+!flowdb/testdata/bin/9910_migrate_test_data.sh
+!flowdb/testdata/bin/migrate_synth_data.py
 !flowdb/testdata/test_data/Pipfile
 !flowdb/testdata/test_data/Pipfile.lock
 !flowdb/testdata/test_data/sql/