feat: add dags ci and cd github actions

.github/workflows/dags_deploy.yml

@@ -0,0 +1,28 @@
+name: Sync DAGs
+
+on:
+ workflow_run:
+ workflows:
+ - 'DAGs Check'
+ types:
+ - completed
+ pull_request:
+ types:
+ - closed
+
+jobs:
+ deploy:
+ runs-on: ubuntu-latest
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
+ steps:
+ - uses: actions/checkout@master
+ - uses: jakejarvis/s3-sync-action@master
+ with:
+ args: --exclude "*" --include "*dags.py"
+ env:
+ AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_REGION: 'us-west-2'
+ SOURCE_DIR: 'dags'
+ DEST_DIR: 'dags'

.github/workflows/dags_test.yml

@@ -0,0 +1,36 @@
+name: Dags Check
+on:
+ push:
+ paths:
+ - 'dags/**'
+ pull_request:
+ branches:
+ - main
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: "3.7"
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements/requirements.txt
+ pip check
+ - name: Lint with Flake8
+ run: |
+ pip install flake8
+ flake8 dags --benchmark -v
+ - name: Confirm Black code compliance
+ run: |
+ pip install black
+ black dags -v
+ - name: Test with Pytest
+ run: |
+ pip install pytest
+ cd tests || exit
+ pytest tests.py -v

dags/.airflowignore

@@ -0,0 +1,2 @@
+tests.py
+README.md

dags/tests.py

@@ -0,0 +1,15 @@
+from unittest import TestCase
+
+from airflow.models import DagBag
+
+from . import rikolti_harvest_collection_dag as harvest_dag
+
+DAGS_FOLDER = "."
+
+
+class HarvestDagsTest(TestCase):
+ def dag_bag(self):
+ return DagBag(dag_folder=DAGS_FOLDER, include_examples=False)
+
+ def test_no_import_errors(dag_bag):
+ assert not dag_bag.import_errors