first commit

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.github/ISSUE_TEMPLATE/notifica--o-de-problema.md

@@ -0,0 +1,31 @@
+---
+name: Notificação de problema
+about: Notifique um problema que está ocorrendo para que possamos corrigí-lo.
+title: "[BUG]"
+labels: bug
+assignees: 
+---
+
+**Descreva o problema**
+Uma descrição clara e concisa do problema.
+
+**Passos realizados**
+O passo-a-passo realizado, em detalhes, para chegar a esse problema. Deve ser detalhado a ponto de qualquer pessoa conseguir reproduzir:
+
+1. Vá até '...'
+2. Clique em '....'
+3. Role a tela até '....'
+4. Seu erro estará em '...'
+
+**Comportamento esperado**
+Uma descrição clara e concisa do que você esperava que acontecesse ao invés do problema.
+
+**Capturas de tela**
+Se cabível, inserir capturas de tela relacionadas ao problema.
+
+**Contexto:**
+
+- Sistema operacional: [ex. Ubuntu 20.04]
+- Browser (se aplicável) [ex. chrome, safari]
+- ...
+- Qualquer outra informação de contexto relevante (ex. versão do Python...)

.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md

@@ -0,0 +1,19 @@
+---
+name: Requerimento de funcionalidade
+about: Sugira novas funcionalidades que podem ser úteis a todos os desenvolvedores.
+title: "[FEATURE]"
+labels: enhancement
+assignees: 
+---
+
+**Essa funcionalidade está relacionada a um problema?**
+Em caso afirmativo, descreva o problema.
+
+**Descreva a solução ideal para você**
+Uma descrição de como seria o funcionamento após a implementação dessa funcionalidade sugerida.
+
+**Descreva alternativas que você considerou**
+Antes de chegar à conclusão de que essa funcionalidade deveria ser agregada, você possivelmente cogitou outras alternativas. Descreva-as.
+
+**Contexto adicional**
+Qualquer outra informação relevante.

.github/workflows/build_docker.yaml

@@ -0,0 +1,48 @@
+name: Build Docker image
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - ".github/workflows/cd.yaml"
+ - "pipelines/**/*"
+ - "pyproject.toml"
+ - "Dockerfile"
+ pull_request:
+ branches:
+ - main
+ paths:
+ - ".github/workflows/cd_staging.yaml"
+ - "pipelines/**/*"
+ - "pyproject.toml"
+ - "Dockerfile"
+
+jobs:
+ build-container:
+ name: Build Docker image
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@v1
+ with:
+ registry: ghcr.io
+ username: ${{ github.repository_owner }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Build and push image
+ uses: docker/build-push-action@v2
+ with:
+ context: .
+ file: ./Dockerfile
+ push: true
+ tags: |
+ ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:${{ github.sha }}
+ labels: |
+ org.opencontainers.image.source=${{ github.event.repository.html_url }}
+ org.opencontainers.image.revision=${{ github.sha }}
+ build-args: |
+ BUILDKIT_INLINE_CACHE=1

.github/workflows/cd.yaml

@@ -0,0 +1,74 @@
+name: Register flows (production)
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - ".github/workflows/cd.yaml"
+ - "pipelines/**/*"
+ - "pyproject.toml"
+ - "Dockerfile"
+
+env:
+ PREFECT__BACKEND: cloud
+ PREFECT__CLOUD__API: ${{ secrets.PREFECT__CLOUD__API }}
+ PREFECT__CLOUD__PORT: ${{ secrets.PREFECT__CLOUD__PORT }}
+ PREFECT__SERVER__PROJECT: ${{ secrets.PREFECT__SERVER__PROJECT__PROD }}
+ PREFECT_AUTH_TOML: ${{ secrets.PREFECT_AUTH_TOML }}
+
+jobs:
+ build-container:
+ name: Register flows (production)
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Setup Python version
+ uses: actions/setup-python@v2
+ with:
+ python-version: "3.10"
+
+ - name: Setup Google Cloud credentials
+ uses: google-github-actions/[email protected]
+ with:
+ service_account_key: ${{ secrets.GCP_SA_KEY }}
+ project_id: ${{ secrets.GCP_PROJECT_ID }}
+ export_default_credentials: true
+
+ - name: Install Python dependencies for deploying
+ run: |-
+ pip install -U pip poetry
+ poetry config virtualenvs.create false
+ poetry install --with dev --with ci
+
+ - name: Update image tag in constants
+ run: |-
+ python .github/workflows/scripts/replace_docker_tag.py ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} ${{ github.sha }}
+
+ - name: Get changed files for code tree analysis
+ id: files
+ uses: Ana06/[email protected]
+
+ - name: Perform code tree analysis
+ id: code-tree-analysis
+ continue-on-error: true
+ run: |
+ python .github/workflows/scripts/code_tree_analysis.py "${{ steps.files.outputs.all }}" --write-to-file
+
+ - name: Write auth.toml
+ run: |-
+ mkdir -p $HOME/.prefect
+ echo $PREFECT_AUTH_TOML | base64 --decode > $HOME/.prefect/auth.toml
+
+ - name: Wait for Docker image to be available
+ uses: lewagon/[email protected]
+ with:
+ ref: ${{ github.ref }}
+ check-name: 'Build Docker image'
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ verbose: true
+
+ - name: Register Prefect flows
+ run: |-
+ python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/ --schedule --filter-affected-flows

.github/workflows/cd_staging.yaml

@@ -0,0 +1,76 @@
+name: Register flows (staging)
+
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - ".github/workflows/cd_staging.yaml"
+ - "pipelines/**/*"
+ - "pyproject.toml"
+ - "Dockerfile"
+
+env:
+ PREFECT__BACKEND: cloud
+ PREFECT__CLOUD__API: ${{ secrets.PREFECT__CLOUD__API }}
+ PREFECT__CLOUD__PORT: ${{ secrets.PREFECT__CLOUD__PORT }}
+ PREFECT__SERVER__PROJECT: ${{ secrets.PREFECT__SERVER__PROJECT__STAGING }}
+ PREFECT_AUTH_TOML: ${{ secrets.PREFECT_AUTH_TOML }}
+
+jobs:
+ build-container:
+ if: startsWith(github.head_ref, 'staging/') || contains( github.event.pull_request.labels.*.name, 'staging')
+ name: Register flows (staging)
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Setup Python version
+ uses: actions/setup-python@v2
+ with:
+ python-version: "3.10"
+
+ - name: Setup Google Cloud credentials
+ uses: google-github-actions/[email protected]
+ with:
+ service_account_key: ${{ secrets.GCP_SA_KEY }}
+ project_id: ${{ secrets.GCP_PROJECT_ID }}
+ export_default_credentials: true
+
+ - name: Install Python dependencies for deploying
+ run: |-
+ pip install -U pip poetry
+ poetry config virtualenvs.create false
+ poetry install --with dev --with ci
+
+ - name: Update image tag in constants
+ run: |-
+ python .github/workflows/scripts/replace_docker_tag.py ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} ${{ github.sha }}
+
+ - name: Get changed files for code tree analysis
+ id: files
+ uses: Ana06/[email protected]
+
+ - name: Perform code tree analysis
+ id: code-tree-analysis
+ continue-on-error: true
+ run: |
+ python .github/workflows/scripts/code_tree_analysis.py "${{ steps.files.outputs.all }}" --write-to-file
+
+ - name: Write auth.toml
+ run: |-
+ mkdir -p $HOME/.prefect
+ echo $PREFECT_AUTH_TOML | base64 --decode > $HOME/.prefect/auth.toml
+
+ - name: Wait for Docker image to be available
+ uses: lewagon/[email protected]
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ check-name: 'Build Docker image'
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ verbose: true
+
+ - name: Register Prefect flows
+ run: |-
+ python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/ --no-schedule --filter-affected-flows

.github/workflows/code-tree-analysis.yaml

@@ -0,0 +1,49 @@
+name: CI
+
+on:
+ pull_request:
+
+jobs:
+ code-tree-analysis:
+ name: Code tree analysis
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Setup Python version
+ uses: actions/setup-python@v2
+ with:
+ python-version: "3.10"
+
+ - name: Install Python dependencies for deploying
+ run: |-
+ pip install -U pip poetry
+ poetry config virtualenvs.create false
+ poetry install --with dev --with ci
+
+ - name: Check if `prefect build` works
+ run: |
+ prefect build
+
+ - name: Get changed files for code tree analysis
+ id: files
+ uses: Ana06/[email protected]
+
+ - name: Perform code tree analysis
+ id: code-tree-analysis
+ continue-on-error: true
+ run: |
+ python .github/workflows/scripts/code_tree_analysis.py "${{ steps.files.outputs.all }}"
+
+ - name: Delete previous comments
+ uses: izhangzhihao/delete-comment@master
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ delete_user_name: github-actions[bot]
+ issue_number: ${{ github.event.number }} # remove comments from the current PR
+
+ - name: Comment PR
+ uses: thollander/actions-comment-pull-request@v1
+ with:
+ message: "${{ steps.code-tree-analysis.outputs.pr-message }}"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint.yaml

@@ -0,0 +1,34 @@
+name: CI
+
+on:
+ push:
+
+jobs:
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5
+
+ - name: Lint Dockerfile
+ uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf
+ with:
+ dockerfile: Dockerfile
+
+ - name: Set up Python
+ uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e
+ with:
+ python-version: "3.10"
+
+ - name: Set up Poetry and upgrade pip
+ run: |
+ pip install -U pip poetry
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.create false && poetry install --with dev --with ci
+
+ - name: Lint with black, isort and flake8
+ run: |
+ task lint