New build for tesoro

.github/workflows/docker-image.yml

@@ -1,61 +1,227 @@
-name: Docker Build and Push
-on: [push, pull_request]
+name: Test, Build and Publish docker image
+run-name: Docker Build for ${{ github.actor }} on branch ${{ github.ref_name }}
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+on: 
+  push:
+    branches:
+      - master
+      - main
+      - test/*
+    paths-ignore:
+      - 'docs/**'
+      - 'requirements.docs.txt'
+      - 'mkdocs.yml'
+      - 'CNAME'
+      - 'Dockerfile.docs'
+
+  release:
+    types: [created]
+
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'requirements.docs.txt'
+      - 'mkdocs.yml'
+      - 'CNAME'
+      - 'Dockerfile.docs'
+
+
 jobs:
+  lint:
+    name: linter
+    runs-on: ubuntu-latest
+    if: success() || failure()  # Continue running if other jobs fail
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - uses: psf/black@main
+
+  test:
+    name: python ${{ matrix.python-version }} tests
+    runs-on: ubuntu-latest
+    if: success() || failure()   # Continue running if other jobs fail
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9]
+
+    steps:
+      - name: Checkout recursively
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install testing dependencies
+        run: |
+          pip3 instal kapitan
+          pip3 install -r requirements.txt
+
+      - name: Run tests
+        run: |-
+          make test
   build:
+    name: build ${{ matrix.platform }} image
+    if: success() || failure()   # Continue running if other jobs fail
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
       - name: Checkout tesoro recursively
-        uses: actions/checkout@master
+        uses: actions/checkout@v4
         with:
-          submodules: 'recursive'
-      - name: Strip git ref prefix from tag version and store in REF_NAME
+          submodules: recursive
+
+      # Setup QEMU and Buildx to build multi-platform image
+      # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Builds docker image and allow scoped caching
+      - name: build Tesoro Image
+        uses: docker/build-push-action@v5
+        with:
+          push: False 
+          platforms: ${{ matrix.platform }}
+          load: True
+          file: Dockerfile
+          tags: local-test-${{ matrix.platform }}
+          cache-from: type=gha,scope=$GITHUB_REF_NAME-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-${{ matrix.platform }}
+
+      - name: Test Tesoro for ${{ matrix.platform }}
         run: |
-          echo "TAG_VERSION=${GITHUB_REF#refs/*/v}" >> $GITHUB_ENV
-          echo "REF_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          docker run -t --rm local-test-${{ matrix.platform }} -h
           
-      - name: Strip full version and just keep major part in MAJOR_VERSION VAR
-        run: |
-          echo "MAJOR_VERSION=${TAG_VERSION:0:4}" >> $GITHUB_ENV
-      # Printing versions needs to be a separate step,
-      # as they aren't set during the previous two steps
-      - name: Print Versions
-        run: |
-          echo ${{ env.TAG_VERSION }}
-          echo ${{ env.MAJOR_VERSION }}
-          echo ${{ env.REF_NAME }}
-      - name: "Build PR/versioned tags"
-        if: github.ref != 'refs/heads/master'
-        uses: docker/build-push-action@v1
+
+  publish:
+    name: publish platform images
+    # Only starts if everything else is successful
+    needs: [lint, test, build]
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+    steps:
+      - name: Checkout tesoro recursively
+        uses: actions/checkout@v4
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-          repository: kapicorp/tesoro
-          add_git_labels: true
-          tags: ${{ format('{0}', env.REF_NAME ) }}
-          push: ${{ github.event_name != 'pull_request' }} # push image only on non-pull_requests
-          dockerfile: Dockerfile
-      - name: "Build latest tag"
-        uses: docker/build-push-action@v1
-        if: github.ref == 'refs/heads/master'
+          submodules: recursive
+
+      # Setup QEMU and Buildx to build multi-platform image
+      # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
+        if: env.DOCKERHUB_USERNAME != null
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-          repository: kapicorp/tesoro
-          add_git_labels: true
-          tag_with_ref: true
-          dockerfile: Dockerfile
-      - name: "Test Dockerfile in current ref"
-        run: |
-            [ ${{ env.REF_NAME }} == "master" ] && tagname="latest" || tagname=${{ env.REF_NAME }}
-            docker run -t --rm kapicorp/tesoro:${tagname} -h
-      - name: "Build major version tag"
-        uses: docker/build-push-action@v1
-        if: startsWith(github.ref, 'refs/tags/')
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            name=${{ vars.DOCKERHUB_REPOSITORY }}/tesoro
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+          flavor: |
+            suffix=-${{ matrix.platform }}
+
+      - name: Build and push by digest
+        id: push-digest
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
+        if: env.DOCKERHUB_USERNAME != null
+        uses: docker/build-push-action@v5
+        with:
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{steps.meta.output.labels}}
+          cache-from: type=gha,scope=$GITHUB_REF_NAME-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-${{ matrix.platform }}
+
+  build-multi-architecture:
+    name: combine platform images
+    needs:
+      - publish
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      # Setup QEMU and Buildx to build multi-platform image
+      # This was inspired by this example : https://docs.docker.com/build/ci/github-actions/examples/#multi-platform-images
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
+        if: env.DOCKERHUB_USERNAME != null
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-          repository: kapicorp/tesoro
-          add_git_labels: true
-          tags: ${{ format('{0}', env.MAJOR_VERSION ) }}
-          dockerfile: Dockerfile
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME}}
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            name=${{ vars.DOCKERHUB_REPOSITORY }}/tesoro
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            
+      - uses: int128/docker-manifest-create-action@v1
+        with:
+          tags: ${{ steps.meta.outputs.tags }}
+          builder: buildx
+          suffixes: |
+            -linux-amd64
+            -linux-arm64

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+    - id: black

Dockerfile

@@ -1,4 +1,4 @@
-FROM deepmind/kapitan:0.29
+FROM kapicorp/kapitan
 
 USER root
 WORKDIR /opt/venv/

tests/test_tranform.py

@@ -14,9 +14,14 @@ def test_prepare_obj_k8s_secret(self):
         k8s_obj = {
             "apiVersion": "v1",
             "kind": "Secret",
-            "metadata": {"name": "some-secret", "labels": {"tesoro.kapicorp.com": "enabled"},},
+            "metadata": {
+                "name": "some-secret",
+                "labels": {"tesoro.kapicorp.com": "enabled"},
+            },
             "type": "Opaque",
-            "data": {"file1": b64encode(bytes(ref_tag.encode())),},
+            "data": {
+                "file1": b64encode(bytes(ref_tag.encode())),
+            },
         }
         transformations = prepare_obj("request_uid", k8s_obj)
 
@@ -44,9 +49,14 @@ def test_transform_obj_k8s_secret_original_encoding(self):
         k8s_obj = {
             "apiVersion": "v1",
             "kind": "Secret",
-            "metadata": {"name": "some-secret", "labels": {"tesoro.kapicorp.com": "enabled"},},
+            "metadata": {
+                "name": "some-secret",
+                "labels": {"tesoro.kapicorp.com": "enabled"},
+            },
             "type": "Opaque",
-            "data": {"file1": b64encode(bytes(ref_tag.encode())),},
+            "data": {
+                "file1": b64encode(bytes(ref_tag.encode())),
+            },
         }
         transformations = prepare_obj("request_uid", k8s_obj)
         # reveal base64_ref
@@ -67,9 +77,14 @@ def test_transform_obj_k8s_secret_base64_encoding(self):
         k8s_obj = {
             "apiVersion": "v1",
             "kind": "Secret",
-            "metadata": {"name": "some-secret", "labels": {"tesoro.kapicorp.com": "enabled"},},
+            "metadata": {
+                "name": "some-secret",
+                "labels": {"tesoro.kapicorp.com": "enabled"},
+            },
             "type": "Opaque",
-            "data": {"file1": b64encode(bytes(ref_tag.encode())),},
+            "data": {
+                "file1": b64encode(bytes(ref_tag.encode())),
+            },
         }
         transformations = prepare_obj("request_uid", k8s_obj)
         # reveal base64_ref