Data Pipeline - Bucket 5 semiannually Prod

Data Pipeline - Bucket 5 semiannually Prod #11

Workflow file for this run

.github/workflows/pipeline-bucket5-SA-datasets-prod.yml at 8b94a67

	# The datasets included in this bucket are airports, civic facilities, clinics,
	# customs ports of entry, diagnostic facilities, emergency social service facilities,
	# hospitals, laboratory services, pharmacies,
	# port and terminal, public library, schools, servicebc locations, timber facilities

	name: Data Pipeline - Bucket 5 semiannually Prod
	on:
	schedule:
	- cron: "0 0 1 /6 "
	workflow_dispatch:
	jobs:
	run-airports:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_airports' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline
	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-civic-facilities:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_civic_facilities' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline
	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-clinics:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_clinics' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline
	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-customs-port-of-entry:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_customs_ports_of_entry' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline
	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-diagnostic-facilities:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_diagnostic_facilities' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline
	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-emergency-social-service-facilities:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_emergency_social_service_facilities' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline
	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-hospitals:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_hospitals' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-laboratory-services:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_laboratory_service' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi


	run-local-govt-offices:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_local_govt_offices' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-pharmacies:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_pharmacies' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-port-and-terminal:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_port_and_terminal' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-public-library:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_public_library' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-schools:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_schools' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-servicebc-locations:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_servicebc_locations' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi
	run-timber-facilities:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_5_timber_facilities' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline


	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Data Pipeline - Bucket 5 semiannually Prod #11

Workflow file

Data Pipeline - Bucket 5 semiannually Prod #11

Jobs

Run details

Workflow file for this run