Data Pipeline - Bucket 2 semiannually Prod

Data Pipeline - Bucket 2 semiannually Prod #11

Workflow file for this run

.github/workflows/pipeline-bucket2-SA-datasets-prod.yml at 8b94a67

	# This workflow will build a new container semiannually running the datasets in Bucket 2 semiannually
	# The datasets included in this bucket are, housing, municipalities, NBDPHHSpeeds,
	# Northern Rockies Census Division, phdemographic, regional districts, tourism region, tsunami zones

	name: Data Pipeline - Bucket 2 semiannually Prod
	on:
	schedule:
	- cron: "0 0 1 /6 "
	workflow_dispatch:

	jobs:
	run-municipal-land-title-transfers:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_municipal_land_title_transfers' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi


	run-small-business-count-by-csd:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_small_businesses' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-core-housing-need-by-csd:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_core_housing_need' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-businesses-by-csd:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_businesses_by_csd' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-housing:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_housing' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-municipal-tax-rates:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_municipal_tax_rates' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-municipalities:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_municipalities' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-nbd-phh-speeds:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_nbd_phh_speeds' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-phdeomographic:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_phdemographic' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-regional-districts:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_regional_districts' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-school-districts:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_school_districts' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-tourism-region:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_tourism_region' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-tsunami-zones:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_tsunami_zones' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

	run-wildfire-zones:
	runs-on: ubuntu-latest
	steps:
	# checkout the repo
	- name: "Checkout GitHub Action"
	uses: actions/checkout@master

	- name: "Generate env file"
	run: \|
	echo 'POSTGRES_DB = ${{ secrets.POSTGRES_DB_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_USER = ${{ secrets.POSTGRES_DJANGO_USER_PROD }}' >> .env
	echo 'POSTGRES_DJANGO_PASSWORD = ${{ secrets.POSTGRES_DJANGO_PASSWORD_PROD }}' >> .env
	echo 'POSTGRES_HOST = ${{ secrets.POSTGRES_HOST_PROD }}' >> .env
	echo 'ROUTE_PLANNER_API_KEY = ${{ secrets.ROUTE_PLANNER_API_KEY }}' >> .env
	echo 'BUCKET_COMMAND = bucket_2_wildfire_zones' >> .env

	- name: Start containers
	run: docker-compose -f cit-api/docker-compose.yml up --build cit-api-pipeline

	# docker-compose will return 0 even if python container returns non-zero or crashes
	# ensure job fails if python container returns non-zero code or crashes
	- name: check Container Status
	run: \|
	container_status=$(docker ps --filter 'status=exited' --format '{{.Status}}')
	if [[ $container_status == "Exited (0)" ]];
	then
	echo "Container exited successfully"
	else
	echo "Container exited with error: $container_status"
	exit 1
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Data Pipeline - Bucket 2 semiannually Prod #11

Workflow file

Data Pipeline - Bucket 2 semiannually Prod #11

Jobs

Run details

Workflow file for this run