.gitlab-ci.yml

stages:
  - lint
  - tests
  - sonarcloud
  - deploy pages
  - compile_dbt
  - generate_dbt_dag
  - commit_dbt_dag

variables:
  SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache
  GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
  DBT_PROJECT_DIR: "${CI_PROJECT_DIR}/plugins/dbt_pg_project"
  AIRFLOW_DBT_PROJECT_DIR: "/opt/airflow/dags-config/repo/plugins/dbt_pg_project"
  AIRFLOW_DBT_PROFILE_DIR: "/opt/airflow/dags-config/repo/plugins/dbt_pg_project"
  PYTHON_SCRIPT: "${CI_PROJECT_DIR}/automation/generate_dbt_dag.py"
  DAG_REPO_PATH: "${CI_PROJECT_DIR}/dags/"
  GIT_CI_USER: "CI Bot"
  GIT_CI_EMAIL: "ci@lappis.rocks"

# <------------- Defining Anchors ------------->

default:
  image: python:3.10-slim
  retry: 0
  before_script:
    - python -m pip install --upgrade pip setuptools wheel  # Upgrade pip and related tools
    - pip install -r requirements.txt  # Install dependencies
    - pip install -r requirements.testing.txt  # Install testing dependencies

.default_rule_python: &default_rule_python
  only:
    - merge_requests
    - branches

.default_pytest: &default_pytest
  <<: *default_rule_python
  script:
    - pytest . --junitxml=pytest_report.xml
  coverage: '/^TOTAL.+?(\d+\%)$/'
  artifacts:
    when: always
    paths:
      - pytest_report.xml
    reports:
      junit: pytest_report.xml

.setup_dbt: &setup_dbt
  before_script:
    - apt-get update
    - apt-get install -y wireguard iproute2
    - echo -e "[Interface]\nAddress = 192.168.200.10/32\nSaveConfig = true\nListenPort = 51760\nPrivateKey = $WIREGUARD_PRIVATE_KEY\n\n[Peer]\nPublicKey = $WIREGUARD_PEER_KEY\nAllowedIPs = 192.168.3.0/27\nEndpoint = 200.152.47.48:51820\nPersistentKeepalive = 15\n" > /etc/wireguard/wg0.conf
    - wg-quick up wg0
    - pip install dbt-postgres

# # <------------- Linters ------------->

lint-ruff:
  stage: lint
  <<: *default_rule_python
  script:
    - pip install ruff  # Ensure ruff is installed
    - ruff check .

lint-black:
  stage: lint
  <<: *default_rule_python
  script:
    - pip install black
    - black . --check --verbose --color

lint-markdown:
  stage: lint
  image: node:20-bullseye
  inherit:
    default: false
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - changes:
        - ./**/*.md
  variables:
    SHOW_ERRORS: "true"
  script:
    - echo "---------- Stage 1 - Install Prerequisites ----------"
    - apt-get update
    - apt-get install -y curl
    - npm install -g markdownlint-cli

    - echo "---------- Stage 2 - Run markdownlint (v0.32.2) ----------"
    - bash -c "$(curl -fsSL https://raw.githubusercontent.com/CICDToolbox/markdown-lint/master/pipeline.sh)"

# <------------- Testing ------------->

pytest-3.9:
  stage: tests
  image: python:3.9-slim
  <<: *default_pytest

pytest-3.10:
  stage: tests
  image: python:3.10-slim
  <<: *default_pytest

pytest-3.11:
  stage: tests
  image: python:3.11-slim
  <<: *default_pytest

# <------------- QA ------------->

sonarcloud-check:
  stage: sonarcloud
  image:
    name: sonarsource/sonar-scanner-cli:latest
    entrypoint: [""]
  inherit:
    default: false
  dependencies:
    - pytest-3.9
    - pytest-3.10
    - pytest-3.11
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .sonar/cache
  script:
    - sonar-scanner -X
  only:
    - merge_requests
    - main
    - development


# <------------- Generate dbt DAG ------------->
compile_dbt_project:
  stage: compile_dbt
  image: python:3.9
  <<: *setup_dbt
  inherit:
    default: false
  script:
    - cd $DBT_PROJECT_DIR
    - dbt deps
    - dbt compile
    - wg-quick down wg0
  artifacts:
    paths:
      - $DBT_PROJECT_DIR/target/manifest.json
  only:
    - main
    - development

pages:
  image: python:3.9
  stage: deploy pages
  <<: *setup_dbt
  only:
    refs:
      - main
  script:
    - cd $DBT_PROJECT_DIR
    - dbt deps
    - dbt docs generate
    - mv ./target ${CI_PROJECT_DIR}/public
    - wg-quick down wg0
  artifacts:
    paths:
      - public

process_manifest:
  stage: generate_dbt_dag
  image: python:3.9
  script:
    - python $PYTHON_SCRIPT --manifest_path $DBT_PROJECT_DIR/target/manifest.json --project_path $AIRFLOW_DBT_PROJECT_DIR --profile_path $AIRFLOW_DBT_PROFILE_DIR --dag_folder_path dbt/
  artifacts:
    paths:
      - dbt/*.py
  only:
    - main
    - development

commit_generated_file:
  stage: commit_dbt_dag
  image: python:3.9
  script:
    - git config --global user.email "$GIT_CI_EMAIL"
    - git config --global user.name "$GIT_CI_USER"
    - git checkout $CI_COMMIT_REF_NAME
    - apt-get update -y
    - apt-get install -y rsync
    - rsync -a --delete dbt/ $DAG_REPO_PATH/dbt/
    - git add $DAG_REPO_PATH
    - git diff-index --quiet HEAD || (git commit -m "automation(dbt_dag) Add airflow DAG from dbt manifest [ci skip]" && git push https://dbtAutomation:$CI_JOB_TOKEN@gitlab.com/lappis-unb/decidimbr/servicos-de-dados/airflow-dags.git $CI_COMMIT_REF_NAME)
  only:
    - main
    - development