From c48ce7700469d2eefff24db5d22ad8c98dd74489 Mon Sep 17 00:00:00 2001 From: Anton Kuraev Date: Fri, 20 Sep 2024 23:19:55 -0700 Subject: [PATCH] Build helm index --- .github/pull_request_template.md | 7 +++ .github/scripts/bump-chart-versions.sh | 19 +++++++ .github/scripts/check_alternatives.py | 22 ++++++++ .github/scripts/check_versions.py | 15 ++++++ .github/workflows/lint-test.yaml | 48 +++++++++++++++++ .github/workflows/pr-delete.yaml | 56 +++++++++++++++++++ .github/workflows/pr-release.yaml | 65 +++++++++++++++++++++++ .github/workflows/release.yaml | 35 ++++++++++++ charts/datahub-executor-worker/Chart.yaml | 7 ++- 9 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100755 .github/scripts/bump-chart-versions.sh create mode 100644 .github/scripts/check_alternatives.py create mode 100644 .github/scripts/check_versions.py create mode 100644 .github/workflows/lint-test.yaml create mode 100644 .github/workflows/pr-delete.yaml create mode 100644 .github/workflows/pr-release.yaml create mode 100644 .github/workflows/release.yaml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..fdc5054 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,7 @@ + + +## Checklist +- [ ] The PR conforms to DataHub's [Contributing Guideline](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) (particularly [Commit Message Format](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md#commit-message-format)) +- [ ] Links to related issues (if applicable) +- [ ] Tests for the changes have been added/updated (if applicable) +- [ ] Docs related to the changes have been added/updated (if applicable) diff --git a/.github/scripts/bump-chart-versions.sh b/.github/scripts/bump-chart-versions.sh new file mode 100755 index 0000000..c12b019 --- /dev/null +++ b/.github/scripts/bump-chart-versions.sh @@ -0,0 +1,19 @@ +#! /bin/sh + +export TAG=$1 + +updateVersion() { + TAG=$1 yq -i e '.version |= env(TAG) ' $2; + echo "Version is updated to $1 in $2" +} + +# updating subcharts +for file in charts/datahub-executor-worker/*/*/Chart.yaml; do + updateVersion $TAG $file +done + +# updating datahub chart +updateVersion $TAG charts/datahub-executor-worker/Chart.yaml + + + yq -i e ".dependencies[].version |= env(TAG)" charts/datahub-executor-worker/Chart.yaml \ No newline at end of file diff --git a/.github/scripts/check_alternatives.py b/.github/scripts/check_alternatives.py new file mode 100644 index 0000000..ca06441 --- /dev/null +++ b/.github/scripts/check_alternatives.py @@ -0,0 +1,22 @@ +import os +import re +import fnmatch + +def find_files(directory, pattern): + matches = [] + for root, _, filenames in os.walk(directory): + for filename in fnmatch.filter(filenames, pattern): + matches.append(os.path.join(root, filename)) + return matches + +directory_to_search = 'charts/datahub-executor-worker/subcharts' +pattern = '*deployment-alternatives*' + +matched_files = find_files(directory_to_search, pattern) +for file in matched_files: + with open(file) as f: + lines = f.readlines() + for line in lines: + if re.compile("{{\s+.Values").search(line): + raise Exception(f"Found problem in {file}: {line}. Please use $.Values. instead of .Values.") + \ No newline at end of file diff --git a/.github/scripts/check_versions.py b/.github/scripts/check_versions.py new file mode 100644 index 0000000..e32c057 --- /dev/null +++ b/.github/scripts/check_versions.py @@ -0,0 +1,15 @@ +import yaml + +DATAHUB_CHART_PATH = 'charts/datahub-executor-worker/Chart.yaml' +with open(DATAHUB_CHART_PATH) as datahub_chart_yaml: + datahub_chart_obj = yaml.safe_load(datahub_chart_yaml) + dependencies = datahub_chart_obj['dependencies'] + for dependency in dependencies: + dependency_version_main_chart = dependency['version'] + repository = dependency['repository'].replace("file://./", "") + subchart_path = DATAHUB_CHART_PATH.replace("Chart.yaml", "") + repository + '/Chart.yaml' + with open(subchart_path) as subchart_yaml: + subchart_obj = yaml.safe_load(subchart_yaml) + subchart_version = subchart_obj['version'] + if dependency_version_main_chart != subchart_version: + raise Exception(f"Version mismatch in {subchart_path}. Main chart: {dependency_version_main_chart}, Subchart: {subchart_version}") \ No newline at end of file diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml new file mode 100644 index 0000000..b7eddf9 --- /dev/null +++ b/.github/workflows/lint-test.yaml @@ -0,0 +1,48 @@ +name: Lint and Test Charts + +on: + push: + branches: + - main + paths-ignore: + - '**.md' + - 'LICENSE' + - 'NOTICE' + pull_request: + paths-ignore: + - '**.md' + - 'LICENSE' + - 'NOTICE' + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.4.0 + + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.0.1 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --target-branch ${{ github.event.pull_request.base.ref }}) + + - name: Run chart-testing (lint) + run: ct lint --target-branch ${{ github.event.pull_request.base.ref }} + + - name: Run deployment alternatives Check + run: | + python3 .github/scripts/check_alternatives.py diff --git a/.github/workflows/pr-delete.yaml b/.github/workflows/pr-delete.yaml new file mode 100644 index 0000000..1cbf593 --- /dev/null +++ b/.github/workflows/pr-delete.yaml @@ -0,0 +1,56 @@ +name: Delete PR Helm Charts + +on: + workflow_dispatch: + inputs: + branch: + description: 'branch name to delete' + required: true + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + pages: write + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: gh-pages + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Extract branch name + shell: bash + run: | + echo "branch=$(echo ${{ github.event.inputs.branch }})" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Download YQ + uses: chrisdickinson/setup-yq@v1.0.1 + with: + yq-version: v4.28.2 + + - name: Debug step + run: | + echo "Branch: ${{ steps.extract_branch.outputs.branch }}" + echo "Chart version: 0.0.0-${{ steps.extract_branch.outputs.branch }}" + cat pr/index.yaml | yq '.entries.datahub-executor-worker[] | select(.version | test("${{ steps.extract_branch.outputs.branch }}"))' + find pr/ -name '*${{ steps.extract_branch.outputs.branch }}*' + + - name: Delete PR Helm charts + run: | + cat pr/index.yaml | yq 'del(.entries.datahub-executor-worker[] | select(.version | test("${{ steps.extract_branch.outputs.branch }}")))' > pr/index.yaml + git status + git ls-files --deleted | xargs git add --all + # Ensure that apiVersion exists + yq -i '.apiVersion = "v1"' pr/index.yaml + git add pr/index.yaml + git status + git commit -m "Deleted all Helm charts from branch: ${{ steps.extract_branch.outputs.branch }}" + git push diff --git a/.github/workflows/pr-release.yaml b/.github/workflows/pr-release.yaml new file mode 100644 index 0000000..f131ef9 --- /dev/null +++ b/.github/workflows/pr-release.yaml @@ -0,0 +1,65 @@ +name: Publish PR Helm Charts + +on: + workflow_dispatch: + push: + branches: + - main + paths-ignore: + - '**.md' + - 'LICENSE' + - 'NOTICE' + pull_request: + paths-ignore: + - '**.md' + - 'LICENSE' + - 'NOTICE' + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + pages: write + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Fetch history + run: git fetch --prune --unshallow + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Extract branch name + shell: bash + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + echo "commit=$(echo ${{ github.sha }} | cut -c -7 )" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Debug step + run: | + echo "Branch: ${{ steps.extract_branch.outputs.branch }}" + echo "Commit: ${{ steps.extract_branch.outputs.commit }}" + echo "Sha: ${{ github.sha }}" + echo "Latest Chart version: 0.0.0-${{ steps.extract_branch.outputs.branch }}" + echo "Specific Chart version: 0.0.0-${{ steps.extract_branch.outputs.branch }}-${{ steps.extract_branch.outputs.commit }}" + + - name: Publish Helm charts for specific commit + uses: stefanprodan/helm-gh-pages@master + with: + charts_url: https://executor-helm.acryl.io + target_dir: pr + chart_version: 0.0.0-${{ steps.extract_branch.outputs.branch }}-${{ steps.extract_branch.outputs.commit }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish Latest PR Helm charts + uses: stefanprodan/helm-gh-pages@master + with: + charts_url: https://executor-helm.acryl.io + target_dir: pr + chart_version: 0.0.0-${{ steps.extract_branch.outputs.branch }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..ad48a3d --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,35 @@ +name: Release Charts + +on: + push: + branches: + - main + paths-ignore: + - '**.md' + - 'LICENSE' + - 'NOTICE' + workflow_dispatch: + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + pages: write + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Fetch history + run: git fetch --prune --unshallow + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Publish Helm charts + uses: stefanprodan/helm-gh-pages@master + with: + charts_url: https://executor-helm.acryl.io + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/charts/datahub-executor-worker/Chart.yaml b/charts/datahub-executor-worker/Chart.yaml index 3757688..6d055db 100644 --- a/charts/datahub-executor-worker/Chart.yaml +++ b/charts/datahub-executor-worker/Chart.yaml @@ -2,5 +2,8 @@ apiVersion: v2 name: datahub-executor-worker description: A Helm chart for datahub-executor-worker type: application -version: 0.0.5 -appVersion: v0.0.1 +version: 0.0.6 +appVersion: 0.0.1 +maintainers: + - name: DataHub + email: datahub@acryl.io