diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..971b794 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,6 @@ +# These are supported funding model platforms + +github: +- fredleger +- neilime +patreon: webofmars diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d3eba5f..df09be8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,8 +3,11 @@ version: 2 updates: - package-ecosystem: "github-actions" - directory: "/" - open-pull-requests-limit: 20 + directories: + - "/" + open-pull-requests-limit: 3 + labels: + - "github_actions" schedule: interval: "weekly" day: friday @@ -12,7 +15,9 @@ updates: - package-ecosystem: "docker" directory: "/" - open-pull-requests-limit: 20 + open-pull-requests-limit: 3 + labels: + - "docker" schedule: interval: "weekly" day: friday @@ -20,7 +25,9 @@ updates: - package-ecosystem: "pip" directory: "/" - open-pull-requests-limit: 20 + open-pull-requests-limit: 3 + labels: + - "pip" schedule: interval: "weekly" day: friday diff --git a/.github/workflows/__shared-ci.yml b/.github/workflows/__shared-ci.yml index b980e9c..cf17ebe 100644 --- a/.github/workflows/__shared-ci.yml +++ b/.github/workflows/__shared-ci.yml @@ -13,9 +13,21 @@ on: jobs: + init: + name: "prepare environment for jobs" + runs-on: self-hosted + steps: + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + shellcheck: name: "Shell: Lint Shell Scripts" runs-on: self-hosted + needs: + - init steps: - name: Checkout uses: actions/checkout@v4 @@ -26,6 +38,8 @@ jobs: hadolint: name: "Docker: Lint Dockerfile" runs-on: self-hosted + needs: + - init steps: - name: Checkout uses: actions/checkout@v4 @@ -36,10 +50,10 @@ jobs: docker-build-images: name: "Docker: Build Images" needs: + - init - shellcheck - hadolint - # FIXME: use a released version of the action when it's ready - uses: hoverkraft-tech/ci-github-container/.github/workflows/docker-build-images.yml@feat/extra-buildx-cache-directives + uses: hoverkraft-tech/ci-github-container/.github/workflows/docker-build-images.yml@0.18.0 permissions: actions: write contents: read @@ -48,15 +62,15 @@ jobs: packages: write pull-requests: read secrets: - oci-registry-password: ${{ secrets.OCI_REGISTRY_PASSWORD }} + oci-registry-password: ${{ secrets.GITHUB_TOKEN }} with: runs-on: '["self-hosted"]' - oci-registry: ${{ vars.OCI_REGISTRY }} - oci-registry-username: ${{ vars.OCI_REGISTRY_USERNAME }} + oci-registry: ghcr.io + oci-registry-username: ${{ github.actor }} images: | [{ "name": "app", - "repository": "${{ vars.OCI_REGISTRY_IMAGE_REPOSITORY }}", + "repository": "${{ github.repository }}", "tag": "${{ inputs.tag }}", "dockerfile": "./Dockerfile", "platforms": [ @@ -66,9 +80,10 @@ jobs: }] chart-testing: - name: "Helm: Chart Testing" + name: "Tests: helm chart" runs-on: self-hosted needs: + - init - docker-build-images steps: - name: Checkout @@ -94,13 +109,13 @@ jobs: run: ct lint --target-branch ${{ github.event.repository.default_branch }} - name: Create kind cluster if: steps.list-changed.outputs.changed == 'true' - uses: helm/kind-action@v1.9.0 + uses: helm/kind-action@v1.10.0 - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' run: | ct install \ --target-branch ${{ github.event.repository.default_branch }} \ - --helm-extra-args "--set image.tag=${{ needs.docker-build-images.outputs.built-images[0].tag }} --wait" + --helm-extra-args "--set image.tag=${{ fromJson(needs.docker-build-images.outputs.built-images).app.tags[0] }} --wait" - name: show pods if: steps.list-changed.outputs.changed == 'true' run: | @@ -109,3 +124,55 @@ jobs: kubectl create job --from=cronjob/ovh-snapshoter -n default ovh-snapshoter-job sleep 10 kubectl get pods -n default + + app-testing: + name: "Tests: application" + runs-on: self-hosted + needs: + - init + - docker-build-images + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Run tests + run: | + echo "OS_PASSWORD length: ${#OS_PASSWORD}" + echo "OS_PROJECT_ID length: ${#OS_PROJECT_ID}" + echo "OS_REGION_NAME length: ${#OS_REGION_NAME}" + echo "OS_TENANT_ID length: ${#OS_TENANT_ID}" + echo "OS_TENANT_NAME length: ${#OS_TENANT_NAME}" + echo "OS_USERNAME length: ${#OS_USERNAME}" + echo "OS_VOLUMES length: ${#OS_VOLUMES}" + docker run --rm -i \ + -e DRY_RUN=true \ + -e CLEANUP=true \ + -e OS_PASSWORD \ + -e OS_PROJECT_ID \ + -e OS_REGION_NAME \ + -e OS_TENANT_ID \ + -e OS_TENANT_NAME \ + -e OS_USERNAME \ + -e OS_VOLUMES \ + ${IMAGE} > output + env: + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} + OS_PROJECT_ID: ${{ secrets.OS_PROJECT_ID }} + OS_REGION_NAME: ${{ secrets.OS_REGION_NAME }} + OS_TENANT_ID: ${{ secrets.OS_TENANT_ID }} + OS_TENANT_NAME: ${{ secrets.OS_TENANT_NAME }} + OS_USERNAME: ${{ secrets.OS_USERNAME }} + OS_VOLUMES: ${{ secrets.OS_VOLUMES }} + IMAGE: ghcr.io/${{ github.repository }}/app:${{ fromJson(needs.docker-build-images.outputs.built-images).app.tags[0] }} + - name: show output + run: | + cat output + - name: check if snapshot was created + run: | + echo -n "check if snapshot was created: " + grep -q "would create snapshot $(date +%Y%m%d)" output && echo 'ok' + - name: check if old snapshot will be deleted + run: | + echo -n "check if old snapshot will be deleted: " + grep -q "would remove snapshot with id=" output && echo ok + - name: delete output file + run: rm output diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.yml index b9af944..076b1e0 100644 --- a/.github/workflows/main-ci.yml +++ b/.github/workflows/main-ci.yml @@ -5,6 +5,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read diff --git a/.github/workflows/merge-group-ci.yml b/.github/workflows/merge-group-ci.yml new file mode 100644 index 0000000..118c003 --- /dev/null +++ b/.github/workflows/merge-group-ci.yml @@ -0,0 +1,35 @@ +name: Pull request - Continuous Integration + +on: + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + pull-request-labeler: + name: Pull request labeler + runs-on: self-hosted + permissions: + contents: write + pull-requests: write + steps: + - uses: release-drafter/release-drafter@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + disable-releaser: true + + ci: + name: Continuous Integration + uses: ./.github/workflows/__shared-ci.yml + permissions: + actions: write + contents: read + id-token: write + issues: read + packages: write + pull-requests: read + secrets: inherit diff --git a/.github/workflows/pull-request-ci.yml b/.github/workflows/pull-request-ci.yml index bf5ee4a..cf90ecd 100644 --- a/.github/workflows/pull-request-ci.yml +++ b/.github/workflows/pull-request-ci.yml @@ -5,6 +5,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: pull-request-labeler: @@ -19,15 +23,3 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: disable-releaser: true - - ci: - name: Continuous Integration - uses: ./.github/workflows/__shared-ci.yml - permissions: - actions: write - contents: read - id-token: write - issues: read - packages: write - pull-requests: read - secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4159d9c..75a994c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,10 @@ name: 🚀 Release on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: check-branches: @@ -51,7 +55,7 @@ jobs: pull-requests: read secrets: inherit - helm-push: + helm-publish: name: "Helm: push chart to OCI registry" needs: [update_release_draft, ci] runs-on: self-hosted @@ -60,20 +64,39 @@ jobs: - uses: actions/checkout@v4 # install tools with asdf - - name: 📦 Install tools with asdf + - name: install tools with asdf uses: asdf-vm/actions/install@v3 - - name: 🔒 Login to OCI registry + - name: login to OCI registry run: | echo "+ login to OCI registry" helm registry login ${OCI_REGISTRY} -u "${OCI_REGISTRY_USERNAME}" -p "${OCI_REGISTRY_PASSWORD}" env: - OCI_REGISTRY: ${{ vars.OCI_REGISTRY }} - OCI_REGISTRY_USERNAME: ${{ vars.OCI_REGISTRY_USERNAME }} - OCI_REGISTRY_PASSWORD: ${{ secrets.OCI_REGISTRY_PASSWORD }} + OCI_REGISTRY: ghcr.io + OCI_REGISTRY_USERNAME: ${{ github.actor }} + OCI_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: update helm chart version + run: | + cd "${CHART_PATH}" + echo "+ update helm chart version" + sed -i "s/version: .*/version: ${VERSION}/g" Chart.yaml + sed -i "s/appVersion: .*/appVersion: ${VERSION}/g" Chart.yaml + env: + CHART_PATH: ${{github.workspace}}/helm/chart + VERSION: ${{ needs.update_release_draft.outputs.latestRelease }} + + - name: commit helm chart version changes + uses: stefanzweifel/git-auto-commit-action@v5 + + - name: update helm chart docs + uses: shaybentk/helm-docs-action@v0.0.1 + with: + working-dir: helm/chart + git-push: "true" # Push the chart - - name: ⚓ Push Helm Chart to OCI registry + - name: push helm chart to OCI registry uses: hoverkraft-tech/helm-push@v5.0.0 with: useOCIRegistry: true @@ -88,7 +111,7 @@ jobs: version: ${{ needs.update_release_draft.outputs.latestRelease }} appVersion: ${{ needs.update_release_draft.outputs.latestRelease }} env: - OCI_REGISTRY: ${{ vars.OCI_REGISTRY }} - OCI_REGISTRY_USERNAME: ${{ vars.OCI_REGISTRY_USERNAME }} - OCI_REGISTRY_PASSWORD: ${{ secrets.OCI_REGISTRY_PASSWORD }} - OCI_REGISTRY_CHART_REPOSITORY: ${{ vars.OCI_REGISTRY_CHART_REPOSITORY }} + OCI_REGISTRY: ghcr.io + OCI_REGISTRY_USERNAME: ${{ github.actor }} + OCI_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + OCI_REGISTRY_CHART_REPOSITORY: 'charts' diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1ccd41c..918f6b7 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,6 +4,10 @@ on: schedule: - cron: "30 1 * * *" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: stale: runs-on: self-hosted diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..62bd47b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-merge-conflict + - id: end-of-file-fixer + - id: check-added-large-files + - id: check-case-conflict + # - id: check-json + - id: check-executables-have-shebangs + - id: check-symlinks + - id: check-yaml + exclude: ^helm/ + - id: detect-aws-credentials + - id: mixed-line-ending + - id: trailing-whitespace + + - repo: https://github.com/gruntwork-io/pre-commit + rev: v0.1.24 + hooks: + - id: helmlint + + - repo: https://github.com/jtyr/kubeconform-helm + rev: v0.1.17 + hooks: + - id: kubeconform-helm + + - repo: https://github.com/norwoodj/helm-docs + rev: v1.14.2 + hooks: + - id: helm-docs + args: + # Make the tool search for charts only under the `charts` directory + - --chart-search-root=helm/chart diff --git a/.tool-versions b/.tool-versions index 379375f..07fbb8f 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,5 @@ -helm 3.12.1 -kubectl 1.28.5 +helm 3.14.3 +kubectl 1.30.5 +helm-ct 3.10.1 +kubeconform 0.6.4 +helm-docs 1.14.2 diff --git a/Dockerfile b/Dockerfile index 1fb7884..11639d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim-bullseye as builder +FROM python:3.12-slim-bullseye AS builder # hadolint ignore=DL3008 RUN set -eux; export DEBIAN_FRONTEND=noninteractive; \ @@ -14,19 +14,21 @@ RUN --mount=type=cache,target=~/.cache/pip \ FROM python:3.12-slim-bullseye -ENV PYTHONUNBUFFERED="1" -ENV OS_AUTH_URL: https://auth.cloud.ovh.net/v3/ -ENV OS_IDENTITY_API_VERSION: "3" -ENV OS_PASSWORD: "" -ENV OS_PROJECT_DOMAIN_NAME: Default -ENV OS_PROJECT_ID: "" -ENV OS_REGION_NAME: GRA5 -ENV OS_TENANT_ID: "" -ENV OS_TENANT_NAME: "" -ENV OS_USER_DOMAIN_NAME: Default -ENV OS_USERNAME: "" -ENV OS_VOLUMES: "" -ENV PATH=/root/.local/bin:$PATH +ENV PYTHONUNBUFFERED="1" \ + DRY_RUN="false" \ + CLEANUP="false" \ + OS_AUTH_URL=https://auth.cloud.ovh.net/v3/ \ + OS_IDENTITY_API_VERSION="3" \ + OS_PASSWORD="" \ + OS_PROJECT_DOMAIN_NAME=Default \ + OS_PROJECT_ID="" \ + OS_REGION_NAME=GRA5 \ + OS_TENANT_ID="" \ + OS_TENANT_NAME="" \ + OS_USER_DOMAIN_NAME=Default \ + OS_USERNAME="" \ + OS_VOLUMES="" \ + PATH=/root/.local/bin:$PATH COPY --from=builder /root/.local /root/.local COPY entrypoint.sh /entrypoint.sh diff --git a/README.md b/README.md index 8cda1f2..aa9de20 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,32 @@ This is why this project was created. enjoy. - if this succeed it will take a backup of all the volumes pointed by the `OS_VOLUMES` var - finaly the script will clean all snapshots marked for cleanup +## install + +```shell +helm upgrade ovh-snapshoter oci://ghcr.io/hoverkraft-tech/ovh-snapshoter/charts \ + --install --create-namespace --namespace ovh-snapshoter \ + --set config.osProjectId=xxxxxx --set config.osUsername=xxxxxx .... +``` + +The mandatory values are the following: + +```yaml +config: + osPassword: "" + osProjectId: "" + osRegionName: GRA11 + osTenantId: "" + osTenantName: "" + osUsername: "" + osVolumes: [] +``` + +You can get all of them (except `osVolumes`) by downloading an horizon config file from OVH UI +`osVolumes` is a list of volume ids that you want to backup on cronjob run (you can get them from OVH public cloud UI or horizon) + +For full documentation check [here](./helm/chart/README.md) + ## contributing - Of course PRs and suggestions are welcome diff --git a/artifacthub-repo.yml b/artifacthub-repo.yml new file mode 100644 index 0000000..cce6414 --- /dev/null +++ b/artifacthub-repo.yml @@ -0,0 +1,17 @@ +# Artifact Hub repository metadata file +# +# Some settings like the verified publisher flag or the ignored packages won't +# be applied until the next time the repository is processed. Please keep in +# mind that the repository won't be processed if it has not changed since the +# last time it was processed. Depending on the repository kind, this is checked +# in a different way. For Helm http based repositories, we consider it has +# changed if the `index.yaml` file changes. For git based repositories, it does +# when the hash of the last commit in the branch you set up changes. This does +# NOT apply to ownership claim operations, which are processed immediately. +# +repositoryID: aa153811-a4aa-43ba-a9c6-ab63f8fe4775 +owners: + - name: Frederic Leger + email: frederic@webofmars.com + - name: Emilien Escalle + email: emilien.escalle@escemi.com diff --git a/helm/chart/README.md b/helm/chart/README.md new file mode 100644 index 0000000..341715b --- /dev/null +++ b/helm/chart/README.md @@ -0,0 +1,47 @@ +# ovh-snapshoter + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square) + +A Helm chart for ovh-snapshoter + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| config.cleanup | string | `"true"` | | +| config.dryRun | string | `"false"` | | +| config.osAuthUrl | string | `"https://auth.cloud.ovh.net/v3"` | | +| config.osIdentityApiVersion | int | `3` | | +| config.osPassword | string | `""` | | +| config.osProjectDomainName | string | `"Default"` | | +| config.osProjectId | string | `""` | | +| config.osRegionName | string | `"GRA11"` | | +| config.osTenantId | string | `""` | | +| config.osTenantName | string | `""` | | +| config.osUserDomainName | string | `"Default"` | | +| config.osUsername | string | `""` | | +| config.osVolumes | list | `[]` | | +| cronjob.schedule | string | `"0 0 * * *"` | | +| deployment.enabled | bool | `false` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.registry | string | `"ghcr.io"` | | +| image.repository | string | `"hoverkraft-tech/ovh-snapshoter/app"` | | +| image.tag | string | `""` | | +| imagePullSecrets | list | `[]` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/helm/chart/values.yaml b/helm/chart/values.yaml index 1742a2b..97e0094 100644 --- a/helm/chart/values.yaml +++ b/helm/chart/values.yaml @@ -5,8 +5,8 @@ replicaCount: 1 image: - registry: harbor.hoverkraft.cloud - repository: ovh-snapshoter/app + registry: ghcr.io + repository: hoverkraft-tech/ovh-snapshoter/app pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" diff --git a/requirements.txt b/requirements.txt index 941216f..0021eae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,16 +2,16 @@ attrs==23.2.0 autopage==0.5.2 certifi==2024.8.30 cffi==1.16.0 -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 cliff==4.6.0 cmd2==2.4.3 -cryptography==42.0.5 +cryptography==43.0.3 debtcollector==3.0.0 decorator==5.1.1 dogpile.cache==1.3.2 gitdb==4.0.11 GitPython==3.1.42 -idna==3.6 +idna==3.7 importlib_metadata==7.1.0 iso8601==2.1.0 jmespath==1.0.1 @@ -40,19 +40,19 @@ python-keystoneclient==5.4.0 python-novaclient==18.6.0 python-openstackclient==6.6.0 python-swiftclient==4.5.0 -pytz==2024.1 +pytz==2024.2 PyYAML==6.0.1 -requests==2.31.0 +requests==2.32.2 requestsexceptions==1.4.0 rfc3986==2.0.0 -setuptools==69.2.0 +setuptools==75.2.0 simplejson==3.19.2 six==1.16.0 smmap==5.0.1 stevedore==5.2.0 tzdata==2024.1 -urllib3==2.2.1 +urllib3==2.2.2 wcwidth==0.2.13 wheel==0.43.0 wrapt==1.16.0 -zipp==3.18.1 +zipp==3.20.2