From 7afa80461c9e507f7d200ccabaeb13f763ea0959 Mon Sep 17 00:00:00 2001
From: Chad Nelson <chadbnelson@gmail.com>
Date: Thu, 27 Jul 2023 11:05:48 -0700
Subject: [PATCH] feat: add dags ci and cd github actions

---
 .github/workflows/dags_deploy.yml      | 28 ++++++++++++++++++++
 .github/workflows/dags_test.yml        | 36 ++++++++++++++++++++++++++
 dags/.airflowignore                    |  2 ++
 dags/README.md                         |  0
 dags/requirements.txt                  |  0
 dags/rikolti_harvest_collection_dag.py |  0
 dags/tests.py                          | 15 +++++++++++
 7 files changed, 81 insertions(+)
 create mode 100644 .github/workflows/dags_deploy.yml
 create mode 100644 .github/workflows/dags_test.yml
 create mode 100644 dags/.airflowignore
 create mode 100644 dags/README.md
 create mode 100644 dags/requirements.txt
 create mode 100644 dags/rikolti_harvest_collection_dag.py
 create mode 100644 dags/tests.py

diff --git a/.github/workflows/dags_deploy.yml b/.github/workflows/dags_deploy.yml
new file mode 100644
index 000000000..5bb927901
--- /dev/null
+++ b/.github/workflows/dags_deploy.yml
@@ -0,0 +1,28 @@
+name: Sync DAGs
+
+on:
+  workflow_run:
+    workflows:
+      - 'DAGs Check'
+    types:
+      - completed
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@master
+      - uses: jakejarvis/s3-sync-action@master
+        with:
+            args: --exclude "*" --include "*dags.py"
+        env:
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: 'us-west-2'
+          SOURCE_DIR: 'dags'
+          DEST_DIR: 'dags'
\ No newline at end of file
diff --git a/.github/workflows/dags_test.yml b/.github/workflows/dags_test.yml
new file mode 100644
index 000000000..266dba13c
--- /dev/null
+++ b/.github/workflows/dags_test.yml
@@ -0,0 +1,36 @@
+name: Dags Check
+on:
+    push:
+        paths:
+        - 'dags/**'
+    pull_request:
+        branches:
+        - main
+jobs:
+    test:
+        runs-on: ubuntu-latest
+        steps:
+        - uses: actions/checkout@v3
+        - name: Set up Python
+          uses: actions/setup-python@v2
+          with:
+            python-version: "3.7"
+        
+        - name: Install dependencies
+          run: |
+            python -m pip install --upgrade pip
+            pip install -r requirements/requirements.txt
+            pip check
+        - name: Lint with Flake8
+          run: |
+            pip install flake8
+            flake8 dags --benchmark -v
+        - name: Confirm Black code compliance
+          run: |
+            pip install black
+            black dags  -v
+        - name: Test with Pytest
+          run: |
+            pip install pytest
+            cd tests || exit
+            pytest tests.py -v
diff --git a/dags/.airflowignore b/dags/.airflowignore
new file mode 100644
index 000000000..746c7c689
--- /dev/null
+++ b/dags/.airflowignore
@@ -0,0 +1,2 @@
+tests.py
+README.md
\ No newline at end of file
diff --git a/dags/README.md b/dags/README.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/dags/requirements.txt b/dags/requirements.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/dags/rikolti_harvest_collection_dag.py b/dags/rikolti_harvest_collection_dag.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/dags/tests.py b/dags/tests.py
new file mode 100644
index 000000000..0f3802f24
--- /dev/null
+++ b/dags/tests.py
@@ -0,0 +1,15 @@
+from unittest import TestCase
+
+from airflow.models import DagBag
+
+from . import rikolti_harvest_collection_dag as harvest_dag
+
+DAGS_FOLDER = "."
+
+
+class HarvestDagsTest(TestCase):
+    def dag_bag(self):
+        return DagBag(dag_folder=DAGS_FOLDER, include_examples=False)
+
+    def test_no_import_errors(dag_bag):
+        assert not dag_bag.import_errors