diff --git a/.github/workflows/deployColl.yml b/.github/workflows/deployColl.yml new file mode 100644 index 0000000..060621e --- /dev/null +++ b/.github/workflows/deployColl.yml @@ -0,0 +1,91 @@ +on: + release: + types: [created] + tags: + - 'coll-v*\.*\.*' + +name: AWS PCM Build and Deploy Collaudo + +jobs: + deploy: + if: ${{ startsWith(github.ref, 'refs/tags/coll') }} + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Print GITHUB_REF + run: echo "GITHUB_REF=${GITHUB_REF}" + - name: Set output + id: vars + run: echo "tag=${GITHUB_REF#refs/*/}" | sed 's/coll-v//' >> "${GITHUB_OUTPUT}" + - name: Check output + env: + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} + run: | + echo $RELEASE_VERSION + echo ${{ steps.vars.outputs.tag }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-south-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push the image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.REPO_NAME }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + # Build a docker container and push it to ECR + docker build --build-arg GEOIP_LICENSE=${{ secrets.GEOIP_LICENSE }} -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + echo "Pushing image to COLL ECR..." + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + echo "Pushed image to COLL ECR " + + - name: Update kube config for coll EKS + id: update-kube-config-coll + run: aws eks update-kubeconfig --name ${{ secrets.EKS_CLUSTER_NAME }} + + - name: Apply the deployment to EKS + id: deploy + env: + IMAGE_TAG: ${{ steps.build-image.outputs.image }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.REPO_NAME }} + run: | + echo "Image tag: $IMAGE_TAG" + tag=$(echo ${IMAGE_TAG##*:}) + echo "coll ecr: $ECR_REGISTRY/$ECR_REPOSITORY:$tag" + echo "Deploying to test EKS..." + cat git-deployment-coll.yml | sed "s|ImagePlaceholder|$ECR_REGISTRY/$ECR_REPOSITORY:$tag|g" | kubectl apply -f - + + - name: Check the deploy to EKS COLL + id: deploy-eks-coll + env: + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + kubectl get cronjobs -n dtd-crawler-coll + tag=$(echo ${IMAGE_TAG##*:}) + for cronjob in $(kubectl get cronjobs | awk '{print $1}' | grep -iv name); do kubectl get cronjob ${cronjob} -o json | jq -r '.spec.jobTemplate.spec.template.spec.containers[].image' | cut -d ':' -f2 | while read result; do { [[ ${result} == ${tag} ]] && echo "Deployment ${cronjob} ok"; } || { echo "Deployment ${cronjob} ko" && exit 1; }; done ; done + + - name: Send SNS notification when the deploy completes in collaudo + id: sns-success + if: success() + run: | + aws sns publish --topic-arn ${{ secrets.SNS_TOPIC_ARN }} --subject "[PCM DTD CRAWLER COLL] Deployment della versione ${{ steps.vars.outputs.tag }} avvenuto con successo" --message "Il deployment della versione ${{ steps.vars.outputs.tag }} è avvenuto con successo su EKS COLL" + + - name: Send SNS notification when the deploy fails in collaudo + id: sns-failure + if: failure() + run: | + aws sns publish --topic-arn ${{ secrets.SNS_TOPIC_ARN }} --subject "[PCM DTD CRAWLER COLL] Deployment della versione ${{ steps.vars.outputs.tag }} fallito" --message "Il deployment della versione ${{ steps.vars.outputs.tag }} è fallito su EKS COLL" diff --git a/.github/workflows/deployProd.yml b/.github/workflows/deployProd.yml new file mode 100644 index 0000000..ee573ae --- /dev/null +++ b/.github/workflows/deployProd.yml @@ -0,0 +1,89 @@ +on: + release: + types: [created] + tags: + - 'prod-v*\.*\.*' + +name: AWS PCM Promote and deploy PROD + +jobs: + deploy: + if: ${{ startsWith(github.ref, 'refs/tags/prod') }} + name: Deploy + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set output + id: vars + run: echo "tag=${GITHUB_REF#refs/*/}" | sed 's/prod-v//' >> "${GITHUB_OUTPUT}" + - name: Check output + env: + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} + run: | + echo $RELEASE_VERSION + echo ${{ steps.vars.outputs.tag }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-south-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push the image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.REPO_NAME }} + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + ECR_REPOSITORY_PROD: ${{ secrets.REPO_NAME_PROD }} + + run: | + docker pull $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY_PROD:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY_PROD:$IMAGE_TAG + + - name: Update kube config for prod EKS + id: update-kube-config-prod + run: aws eks update-kubeconfig --name ${{ secrets.EKS_CLUSTER_NAME_PROD }} + + - name: Apply the deployment to EKS + id: deploy + env: + IMAGE_TAG: ${{ steps.build-image.outputs.image }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.REPO_NAME }} + ECR_REPOSITORY_PROD: ${{ secrets.REPO_NAME_PROD }} + run: | + echo "Image tag: $IMAGE_TAG" + tag=$(echo ${IMAGE_TAG##*:}) + echo "prod ecr: $ECR_REGISTRY/$ECR_REPOSITORY_PROD:$tag" + echo "Deploying to prod EKS..." + cat git-deployment-prod.yml | sed "s|ImagePlaceholder|$ECR_REGISTRY/$ECR_REPOSITORY_PROD:$tag|g" | kubectl apply -f - + + - name: Check the deploy to EKS PROD + id: deploy-eks-prod + env: + IMAGE_TAG: ${{ steps.vars.outputs.tag }} + run: | + kubectl get cronjobs -n dtd-crawler-prod + tag=$(echo ${IMAGE_TAG##*:}) + for cronjob in $(kubectl get cronjobs | awk '{print $1}' | grep -iv name); do kubectl get cronjob ${cronjob} -o json | jq -r '.spec.jobTemplate.spec.template.spec.containers[].image' | cut -d ':' -f2 | while read result; do { [[ ${result} == ${tag} ]] && echo "Deployment ${cronjob} ok"; } || { echo "Deployment ${cronjob} ko" && exit 1; }; done ; done + + - name: Send SNS notification when the deploy completes in production + id: sns-success + if: success() + run: | + aws sns publish --topic-arn ${{ secrets.SNS_TOPIC_ARN }} --subject "[PCM DTD CRAWLER PROD] Deployment della versione ${{ steps.vars.outputs.tag }} avvenuto con successo" --message "Il deployment della versione ${{ steps.vars.outputs.tag }} è avvenuto con successo su EKS PROD" + + - name: Send SNS notification when the deploy fails in production + id: sns-failure + if: failure() + run: | + aws sns publish --topic-arn ${{ secrets.SNS_TOPIC_ARN }} --subject "[PCM DTD CRAWLER PROD] Deployment della versione ${{ steps.vars.outputs.tag }} fallito" --message "Il deployment della versione ${{ steps.vars.outputs.tag }} è fallito su EKS PROD" diff --git a/git-deployment-coll.yml b/git-deployment-coll.yml new file mode 100644 index 0000000..0c15245 --- /dev/null +++ b/git-deployment-coll.yml @@ -0,0 +1,143 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: dtd-crawler-web-server-job-coll + namespace: dtd-crawler-coll + labels: + app: dtd-crawler-web-server-coll +spec: + template: + metadata: + labels: + app: dtd-crawler-web-server-coll + spec: + containers: + - name: dtd-crawler-web-server-coll + image: ImagePlaceholder + imagePullPolicy: Always + command: ["npm", "run", "dist-webserver"] + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-coll + - secretRef: + name: dtd-crawler-database-secret-coll + - secretRef: + name: dtd-crawler-basicauth-secret-coll + - secretRef: + name: dtd-crawler-aws-s3-secret-coll + - secretRef: + name: dtd-crawler-jwt-secret-coll + - secretRef: + name: dtd-crawler-pa2026-secret-coll + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dtd-crawler-pa2026-manager-job-coll + namespace: dtd-crawler-coll + labels: + app: dtd-crawler-pa2026-manager-coll +spec: + schedule: "0 22 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: dtd-crawler-pa2026-manager-coll + spec: + restartPolicy: Never + containers: + - name: dtd-crawler-pa2026-manager-coll + image: ImagePlaceholder + imagePullPolicy: Always + command: ["npm", "run", "dist-PA2026-manager"] + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-coll + - secretRef: + name: dtd-crawler-database-secret-coll + - secretRef: + name: dtd-crawler-basicauth-secret-coll + - secretRef: + name: dtd-crawler-aws-s3-secret-coll + - secretRef: + name: dtd-crawler-jwt-secret-coll + - secretRef: + name: dtd-crawler-pa2026-secret-coll +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dtd-crawler-queue-manager-job-coll + namespace: dtd-crawler-coll + labels: + app: dtd-crawler-queue-manager-coll +spec: + schedule: "0 0,12 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: dtd-crawler-queue-manager-coll + spec: + restartPolicy: Never + containers: + - name: dtd-crawler-queue-manager-coll + image: ImagePlaceholder + imagePullPolicy: Always + command: + - /bin/sh + - -c + - node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/queueManager.js --maxItems 1000 --passedOlderThanDays 0 --failedOlderThanDays 0 --asservationOlderThanDays 0 --manualScanLogic true + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-coll + - secretRef: + name: dtd-crawler-database-secret-coll + - secretRef: + name: dtd-crawler-basicauth-secret-coll + - secretRef: + name: dtd-crawler-aws-s3-secret-coll + - secretRef: + name: dtd-crawler-jwt-secret-coll + - secretRef: + name: dtd-crawler-pa2026-secret-coll +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dtd-crawler-scan-manager-job-coll + namespace: dtd-crawler-coll + labels: + app: dtd-crawler-scan-manager-coll +spec: + schedule: "0 2,14 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: dtd-crawler-scan-manager-coll + spec: + restartPolicy: Never + containers: + - name: dtd-crawler-scan-manager-coll + image: ImagePlaceholder + imagePullPolicy: Always + command: ["npm", "run", "dist-scan-manager"] + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-coll + - secretRef: + name: dtd-crawler-database-secret-coll + - secretRef: + name: dtd-crawler-basicauth-secret-coll + - secretRef: + name: dtd-crawler-aws-s3-secret-coll + - secretRef: + name: dtd-crawler-jwt-secret-coll + - secretRef: + name: dtd-crawler-pa2026-secret-coll \ No newline at end of file diff --git a/git-deployment-prod.yml b/git-deployment-prod.yml new file mode 100644 index 0000000..fde63b4 --- /dev/null +++ b/git-deployment-prod.yml @@ -0,0 +1,143 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: dtd-crawler-web-server-job-prod + namespace: dtd-crawler-prod + labels: + app: dtd-crawler-web-server-prod +spec: + template: + metadata: + labels: + app: dtd-crawler-web-server-prod + spec: + containers: + - name: dtd-crawler-web-server-prod + image: ImagePlaceholder + imagePullPolicy: Always + command: ["npm", "run", "dist-webserver"] + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-prod + - secretRef: + name: dtd-crawler-database-secret-prod + - secretRef: + name: dtd-crawler-basicauth-secret-prod + - secretRef: + name: dtd-crawler-aws-s3-secret-prod + - secretRef: + name: dtd-crawler-jwt-secret-prod + - secretRef: + name: dtd-crawler-pa2026-secret-prod + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dtd-crawler-pa2026-manager-job-prod + namespace: dtd-crawler-prod + labels: + app: dtd-crawler-pa2026-manager-prod +spec: + schedule: "0 22 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: dtd-crawler-pa2026-manager-prod + spec: + restartPolicy: Never + containers: + - name: dtd-crawler-pa2026-manager-prod + image: ImagePlaceholder + imagePullPolicy: Always + command: ["npm", "run", "dist-PA2026-manager"] + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-prod + - secretRef: + name: dtd-crawler-database-secret-prod + - secretRef: + name: dtd-crawler-basicauth-secret-prod + - secretRef: + name: dtd-crawler-aws-s3-secret-prod + - secretRef: + name: dtd-crawler-jwt-secret-prod + - secretRef: + name: dtd-crawler-pa2026-secret-prod +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dtd-crawler-queue-manager-job-prod + namespace: dtd-crawler-prod + labels: + app: dtd-crawler-queue-manager-prod +spec: + schedule: "0 0,12 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: dtd-crawler-queue-manager-prod + spec: + restartPolicy: Never + containers: + - name: dtd-crawler-queue-manager-prod + image: ImagePlaceholder + imagePullPolicy: Always + command: + - /bin/sh + - -c + - node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/queueManager.js --maxItems 1000 --passedOlderThanDays 0 --failedOlderThanDays 0 --asservationOlderThanDays 0 --manualScanLogic true + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-prod + - secretRef: + name: dtd-crawler-database-secret-prod + - secretRef: + name: dtd-crawler-basicauth-secret-prod + - secretRef: + name: dtd-crawler-aws-s3-secret-prod + - secretRef: + name: dtd-crawler-jwt-secret-prod + - secretRef: + name: dtd-crawler-pa2026-secret-prod +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dtd-crawler-scan-manager-job-prod + namespace: dtd-crawler-prod + labels: + app: dtd-crawler-scan-manager-prod +spec: + schedule: "0 2,14 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: dtd-crawler-scan-manager-prod + spec: + restartPolicy: Never + containers: + - name: dtd-crawler-scan-manager-prod + image: ImagePlaceholder + imagePullPolicy: Always + command: ["npm", "run", "dist-scan-manager"] + envFrom: + - configMapRef: + name: dtd-crawler-env-configmap-prod + - secretRef: + name: dtd-crawler-database-secret-prod + - secretRef: + name: dtd-crawler-basicauth-secret-prod + - secretRef: + name: dtd-crawler-aws-s3-secret-prod + - secretRef: + name: dtd-crawler-jwt-secret-prod + - secretRef: + name: dtd-crawler-pa2026-secret-prod \ No newline at end of file