.gitlab-ci.yml

variables:
  PIPELINE_TOOLBOX_IMAGE: "registry.gitlab.com/northern.tech/mender/mender-test-containers:aws-k8s-v1-master"
  EKS_CLUSTER_NAME: "helmci-${CI_PIPELINE_ID}"
  RUN_HELM_CHART_INSTALL:
    value: "false"
    description: "Run Helm Chart tests"
    options:
      - "true"
      - "false"
  ROLE_ARN:
    value: "arn:aws:iam::017659451055:role/nt-mender-helm-eksctl-test"
    description: "AWS IAM role which executes the AWS jobs"
  PUBLISH_ROLE_ARN:
    value: "arn:aws:iam::175683096866:role/nt-helm-chart-s3-publishing"
    description: "AWS IAM role to publish the helm chart to S3"
  GITHUB_HELM_REPO: "github.com/mendersoftware/mender-helm.git"
  SYNC_ENVIRONMENT: "${SYNC_ENVIRONMENT:-staging}"
  CHART_DIR: "mender"
  GITHUB_USER: "mender-test-bot"
  GITHUB_TOKEN: "${GITHUB_BOT_TOKEN_REPO_FULL}"
  MAIN_BRANCH: "master"
  RUN_PLAYGROUND:
    value: "false"
    description: "Run EKS setup playground"
    options:
      - "true"
      - "false"
  PLAYGROUND_EKS_TIMEOUT:
    value: 3600
    description: "Default timeout for the EKS playground setup"
  USE_SPOT_INSTANCES:
    value: "true"
    description: "Use spot instances for cost savings"
    options:
      - "true"
      - "false"

stages:
  - build
  - test
  - publish
  - version-bump

include:
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-check-commits-signoffs.yml'
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-check-license.yml'
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-github-status-updates.yml'
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-check-helm-version-bump.yml'

.set_eks_helmci_vars: &set_eks_helmci_vars
  - export AWS_DEFAULT_REGION=eu-central-1
  - >
    export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s"
    $(aws sts assume-role-with-web-identity
    --role-arn ${ROLE_ARN}
    --role-session-name "GitLabRunner-${CI_PROJECT_ID}-${CI_PIPELINE_ID}"
    --web-identity-token ${AWS_OIDC_TOKEN}
    --duration-seconds ${AWS_OIDC_TOKEN_TIMEOUT:-3600}
    --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]'
    --output text))
  - aws sts get-caller-identity

build:setup_eks_cluster:
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: .pre
  tags:
    - hetzner-amd-beefy
  resource_group: helm
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG || $RUN_PLAYGROUND == "true"'
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      if [[ "${USE_SPOT_INSTANCES}" == "true" ]]; then
        export SPOT_STRING="--spot"
      else
        export SPOT_STRING=""
      fi
  script:
    - |
      echo "INFO - Temporary EKS cluster setup"
      eksctl create cluster \
        --name ${EKS_CLUSTER_NAME} \
        --tags "env=helm-ci-test" \
        --asg-access \
        ${SPOT_STRING} \
        --instance-types c6i.xlarge,m6i.xlarge,m6a.xlarge,t3.xlarge,m5d.xlarge \
        --nodes ${CI_EKS_NUMBER_OF_NODES:-6}
    - echo "INFO - assigning SSO roles for debugging failed tests:"
    - eksctl create iamidentitymapping --cluster ${EKS_CLUSTER_NAME} --arn arn:aws:iam::017659451055:role/AWSReservedSSO_AdministratorAccess_d80c0803237d713a --username "admin:{{SessionName}}" --group "system:masters" --no-duplicate-arns
    - eksctl create iamidentitymapping --cluster ${EKS_CLUSTER_NAME} --arn arn:aws:iam::017659451055:role/AWSReservedSSO_PowerUserAccess_28a0f819b31196fb --username "admin:{{SessionName}}" --group "system:masters" --no-duplicate-arns
    - eksctl create iamidentitymapping --cluster ${EKS_CLUSTER_NAME} --arn arn:aws:iam::017659451055:role/AWSReservedSSO_EKSAccess_8c2574e21d9fc21c --username "admin:{{SessionName}}" --group "system:masters" --no-duplicate-arns
    - eksctl utils associate-iam-oidc-provider --cluster ${EKS_CLUSTER_NAME} --approve
    - eksctl create iamserviceaccount --name ebs-csi-controller-sa --namespace kube-system --cluster ${EKS_CLUSTER_NAME} --role-name AmazonEKS_EBS_CSI_DriverRole --role-only --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy --approve
    - eksctl create iamserviceaccount --name=aws-load-balancer-controller --namespace=kube-system --cluster ${EKS_CLUSTER_NAME} --role-name AmazonEKSLoadBalancerControllerRole --attach-policy-arn=arn:aws:iam::017659451055:policy/AWSLoadBalancerControllerIAMPolicy  --approve
    - helm repo add eks https://aws.github.io/eks-charts
    - helm repo update eks
    - helm install aws-load-balancer-controller eks/aws-load-balancer-controller -n kube-system --set clusterName=${EKS_CLUSTER_NAME} --set serviceAccount.create=false --set serviceAccount.name=aws-load-balancer-controller --wait
    - eksctl create addon --name aws-ebs-csi-driver --cluster ${EKS_CLUSTER_NAME} --service-account-role-arn arn:aws:iam::017659451055:role/AmazonEKS_EBS_CSI_DriverRole --force
    - echo "DEBUG - kubectl cluster version:"
    - kubectl version
    - helm repo add jetstack https://charts.jetstack.io
    - helm install cert-manager jetstack/cert-manager --wait --set installCRDs=true

build:
  stage: build
  image: debian:buster
  tags:
    - hetzner-amd-beefy
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - when: on_success
  before_script:
    - apt-get update -y
    - apt-get install -y curl make
    - curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | DESIRED_VERSION="v3.8.2" bash
  script:
    - make lint
    - make package
  artifacts:
    expire_in: 2w
    paths:
      - mender-*.tgz

.get_kubectl_and_tools: &get_kubectl_and_tools
  # Install kubectl
  - apt update && apt install -yyq curl
  - curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl
  - chmod +x ./kubectl
  - mv ./kubectl /usr/local/bin/kubectl
  # Install AWS CLI and aws-iam-authenticator
  - apt install -yyq awscli
  - curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.15.10/2020-02-22/bin/linux/amd64/aws-iam-authenticator
  - chmod +x ./aws-iam-authenticator
  - mv ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
  # Install kubectx
  - apt install -yyq kubectx

.setup_eks_cluster_staging: &setup_eks_cluster_staging
  # Configure AWS CLI for staging cluster
  - *set_eks_helmci_vars
  - aws eks --region $CI_JOBS_AWS_REGION_STAGING update-kubeconfig --name $CI_JOBS_AWS_EKS_CLUSTER_NAME_STAGING
  - kubectl config set-context --current --namespace=mender-helm-tests

# Skip check license source when the pipeline
# is triggered
test:check-license-source:
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never

test:helm_chart_install:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-setup-test"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - tests/ci-make-deps.sh
    - |
      echo "INFO - installing mender"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - make test
  after_script:
    - echo "INFO - cleaning up to save resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:helm_chart_install_sub_charts:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-setup-test-sub-charts"
    INSTALL_MINIO_DEP_ONLY: "true"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - tests/ci-make-deps.sh $INSTALL_MINIO_DEP_ONLY
    - |
      kubectl apply -f - <<EOS
      apiVersion: cert-manager.io/v1
      kind: Issuer
      metadata:
        name: letsencrypt
      spec:
        acme:
          server: "https://acme-staging-v02.api.letsencrypt.org/directory"
          email: ${LETSENCRYPT_EMAIL:-roberto.giovanardi+letsencrypt@northern.tech}
          privateKeySecretRef:
            name: letsencrypt
          solvers:
          - http01:
              ingress: {}
      EOS
    - |
      echo "INFO - installing mender"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        -f tests/values-helmci-internal-backing-services.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - SKIP_TEARDOWN="true" make test
    - |
      echo "INFO - testing ingress"
      bash tests/ingress-internal-test.sh
    - bash tests/ci-test-teardown.sh || true
  after_script:
    - echo "INFO - cleaning up resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:helm_chart_upgrade_from_previous_lts_to_current_lts:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-upgrade-test-previous-tls"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - tests/ci-make-deps.sh
    - |
      echo "INFO - installing mender tag ${MENDER_PREVIOUS_LTS_RELEASE_TAG:-mender-3.6.5} with helm chart version ${MENDER_LTS_HELM_VERSION:-5.9.0}"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.tag="${MENDER_PREVIOUS_LTS_RELEASE_TAG:-mender-3.6.5}" \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        mender/mender --version ${MENDER_LTS_HELM_VERSION:-5.9.0} || exit 3;
    - bash tests/test-001-wait-for-pods-to-be-ready.sh
    - |
      echo "INFO - installing mender tag ${MENDER_CURRENT_LTS_RELEASE_TAG:-mender-3.7.7}"
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.tag="${MENDER_CURRENT_LTS_RELEASE_TAG:-mender-3.7.7}" \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - helm history mender
    - make test
  after_script:
    - echo "INFO - cleaning up resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:helm_chart_upgrade_from_previous_release_to_current_lts:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-upgrade-test-previous-release"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - tests/ci-make-deps.sh
    - |
      echo "INFO - installing mender tag ${MENDER_PREVIOUS_RELEASE_TAG:-mender-3.4.0} with helm chart version ${MENDER_LTS_HELM_VERSION:-5.0.1}"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.tag="${MENDER_PREVIOUS_RELEASE_TAG:-mender-3.4.0}" \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        mender/mender --version ${MENDER_LTS_HELM_VERSION:-5.0.1} || exit 3;
    - bash tests/test-001-wait-for-pods-to-be-ready.sh
    - |
      echo "INFO - installing mender tag ${MENDER_CURRENT_LTS_RELEASE_TAG:-mender-3.7.7}"
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.tag="${MENDER_CURRENT_LTS_RELEASE_TAG:-mender-3.7.7}" \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - helm history mender
    - make test
  after_script:
    - echo "INFO - cleaning up resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:helm_chart_upgrade_from_current_lts_to_current_release:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  allow_failure: true
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-upgrade-test-current-lts"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - tests/ci-make-deps.sh
    - |
      echo "INFO - installing mender tag ${MENDER_CURRENT_LTS_RELEASE_TAG:-mender-3.7.7}"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.tag="${MENDER_CURRENT_LTS_RELEASE_TAG:-mender-3.7.7}" \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        mender/mender || exit 3;
    - bash tests/test-001-wait-for-pods-to-be-ready.sh
    - |
      echo "INFO - installing mender tag ${MENDER_CURRENT_RELEASE_TAG:-mender-3.7.7}"
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.tag="${MENDER_CURRENT_RELEASE_TAG:-mender-3.7.7}" \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - helm history mender
    - make test
  after_script:
    - echo "INFO - cleaning up resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:kubeconform_tests:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  rules:
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
  script:
    - |
      echo "INFO - running kubeconform tests:"
      while IFS= read -r version; do
        full_version=${version}.0
        echo "INFO - testing version ${full_version}"
        helm template mender/ \
          -f mender/values.yaml \
          -f tests/keys.yaml \
          -f tests/values-helmci.yaml \
          --kube-version ${full_version} \
        | kubeconform --kubernetes-version ${full_version}
      done < <(eksctl version -o json | jq -r '.EKSServerSupportedVersions[]')
    - |
      echo "DEBUG - run failed kubeconform tests: this should fail"
      sed -i 's/apps\/v1/fakeapps\/v42/' mender/templates/gui/deployment.yaml
      while IFS= read -r version; do
        full_version=${version}.0
        echo "INFO - testing version ${full_version}"
        helm template mender/ \
          -f mender/values.yaml \
          -f tests/keys.yaml \
          -f tests/values-helmci.yaml \
          --kube-version ${full_version} \
        | kubeconform --kubernetes-version ${full_version} \
        || echo "INFO - test failed with version ${kubeversion}: this is expected"
      done < <(eksctl version -o json | jq -r '.EKSServerSupportedVersions[]')

test:helm_chart_install_opensource:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-opensource-setup-test"
    INSTALL_MINIO_DEP_ONLY: "true"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - tests/ci-make-deps.sh $INSTALL_MINIO_DEP_ONLY
    - |
      kubectl apply -f - <<EOS
      apiVersion: cert-manager.io/v1
      kind: Issuer
      metadata:
        name: letsencrypt
      spec:
        acme:
          server: "https://acme-staging-v02.api.letsencrypt.org/directory"
          email: ${LETSENCRYPT_EMAIL:-roberto.giovanardi+letsencrypt@northern.tech}
          privateKeySecretRef:
            name: letsencrypt
          solvers:
          - http01:
              ingress: {}
      EOS
    - |
      echo "INFO - installing mender"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-opensource.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${MINIO_accessKey}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${MINIO_secretKey}" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - IS_OPENSOURCE="true" SKIP_TEARDOWN="true" make test
    - |
      echo "INFO - testing ingress"
      bash tests/ingress-internal-test.sh
    - bash tests/ci-test-teardown.sh || true
  after_script:
    - echo "INFO - cleaning up to save resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:helm_chart_install_with_r2:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$RUN_PLAYGROUND == "true"'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG'
  variables:
    NAMESPACE: "mender-setup-test-r2"
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
    - |
      eksctl utils write-kubeconfig --cluster=${EKS_CLUSTER_NAME}
      kubectl create ns "${NAMESPACE}"
      kubectl config set-context --current --namespace="${NAMESPACE}"
    - |
      echo "DEBUG - get kubectl nodes"
      kubectl config current-context
      kubectl get nodes
    - |
      echo "INFO - installing helm from scratch"
      tests/ci-deps-k8s.sh
  script:
    - |
      kubectl apply -f - <<EOS
      apiVersion: cert-manager.io/v1
      kind: Issuer
      metadata:
        name: letsencrypt
      spec:
        acme:
          server: "https://acme-staging-v02.api.letsencrypt.org/directory"
          email: ${LETSENCRYPT_EMAIL:-roberto.giovanardi+letsencrypt@northern.tech}
          privateKeySecretRef:
            name: letsencrypt
          solvers:
          - http01:
              ingress: {}
      EOS
    - |
      echo "INFO - installing mender"
      source ./tests/variables.sh
      helm upgrade -i mender \
        -f mender/values.yaml \
        -f tests/keys.yaml \
        -f tests/values-helmci-internal-backing-services.yaml \
        --wait \
        --timeout=${HELM_UPGRADE_TIMEOUT:-15m} \
        --set global.image.username=${REGISTRY_MENDER_IO_USERNAME} \
        --set global.image.password="${REGISTRY_MENDER_IO_PASSWORD}" \
        --set global.s3.AWS_URI="${R2_TEST_AWS_URI}" \
        --set global.s3.AWS_BUCKET="${R2_TEST_BUCKET}" \
        --set global.s3.AWS_ACCESS_KEY_ID="${R2_TEST_ACCESS_KEY_ID}" \
        --set global.s3.AWS_SECRET_ACCESS_KEY="${R2_TEST_SECRET_ACCESS_KEY}" \
        --set global.s3.AWS_REGION="us-east-1" \
        --namespace "${NAMESPACE}" \
        ./mender || exit 3;
    - SKIP_TEARDOWN="true" make test
    - |
      echo "INFO - testing ingress"
      bash tests/ingress-internal-test.sh
    - bash tests/ci-test-teardown.sh || true
  after_script:
    - echo "INFO - cleaning up resources"
    - helm delete -n "${NAMESPACE}" mender || exit 0 # if it fails it means it's timing out - not that important

test:playground_eks_cluster:
  tags:
    - hetzner-amd-beefy
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: test
  needs: ["build:setup_eks_cluster"]
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: '$RUN_PLAYGROUND == "true"'
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
  script:
    - echo "INFO - you can now play with the EKS cluster for ${PLAYGROUND_EKS_TIMEOUT:-3600} seconds"
    - echo "INFO - please run the following - eksctl utils write-kubeconfig --region ${AWS_DEFAULT_REGION} --cluster ${EKS_CLUSTER_NAME}"
    - sleep ${PLAYGROUND_EKS_TIMEOUT:-3600}

publish:helm_chart_publishing:
  tags:
    - hetzner-amd-beefy
  rules:
    - if: '$CI_COMMIT_TAG'
      changes:
        - mender/Chart.yaml
  stage: publish
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  dependencies:
    - build
  image: debian:buster
  before_script:
    - apt-get update -y
    - apt-get install -y curl make git
    - curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | DESIRED_VERSION="v3.8.2" bash
    - *get_kubectl_and_tools
    - helm plugin install https://github.com/hypnoglow/helm-s3.git --version 0.14.0
    - export AWS_DEFAULT_REGION=us-east-1
    - >
      export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s"
      $(aws sts assume-role-with-web-identity
      --role-arn ${PUBLISH_ROLE_ARN}
      --role-session-name "GitLabRunner-${CI_PROJECT_ID}-${CI_PIPELINE_ID}"
      --web-identity-token ${AWS_OIDC_TOKEN}
      --duration-seconds ${AWS_OIDC_TOKEN_TIMEOUT:-3600}
      --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]'
      --output text))
    - aws sts get-caller-identity
  script:
    - helm repo add mender s3://${S3_HELM_CHART_REPO}
    - helm s3 push --acl="public-read" --relative --timeout=60s ./mender-*.tgz mender
    - aws cloudfront create-invalidation --distribution-id ${S3_HELM_CHART_CDN_DISTRIBUTION_ID} --paths "/*"

.eks_cleanup: &eks_cleanup
  image: ${PIPELINE_TOOLBOX_IMAGE}
  stage: .post
  tags:
    - hetzner-amd-beefy
  resource_group: helm
  allow_failure: true # can't find a way to avoid eks cleanup when no eks is setup, so let's allow to fail for now
  dependencies:
    - build:setup_eks_cluster
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: '$CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "master-next" || $RUN_HELM_CHART_INSTALL == "true" || $CI_COMMIT_TAG || $RUN_PLAYGROUND == "true"'
  id_tokens:
    AWS_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    - *set_eks_helmci_vars
  script:
    - echo "INFO - deleting the temporary EKS cluster"
    - eksctl delete cluster -n ${EKS_CLUSTER_NAME}

cleanup_eks_cluster:failed:
  when: on_failure
  <<: *eks_cleanup

cleanup_eks_cluster:success:
  when: on_success
  <<: *eks_cleanup

cleanup_eks_cluster:failed:manual:
  when: manual
  <<: *eks_cleanup