From 8228f09fae53400a0ac66b79d3161caa6a393af9 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Mon, 11 Sep 2023 12:26:38 +0100 Subject: [PATCH] Sync speciifc repo versions to Test Pulp from Ark --- .../package-sync-version-test-pulp.yml | 47 +++++++++ .../inventory/group_vars/all/test-pulp-repos | 23 ++++- .../test-pulp-repo-specific-version-query.yml | 99 +++++++++++++++++++ ansible/test-pulp-repo-sync.yml | 2 + 4 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/package-sync-version-test-pulp.yml create mode 100644 ansible/test-pulp-repo-specific-version-query.yml diff --git a/.github/workflows/package-sync-version-test-pulp.yml b/.github/workflows/package-sync-version-test-pulp.yml new file mode 100644 index 00000000..577b8879 --- /dev/null +++ b/.github/workflows/package-sync-version-test-pulp.yml @@ -0,0 +1,47 @@ +--- +name: Sync package repository versions to Test Pulp +on: + workflow_dispatch: + inputs: + repo_version: + description: A repository version string maching repository versions to sync from Ark to Test Pulp + type: string + required: false + default: "" + filter: + description: Space-separated list of regular expressions matching short_name of repositories to sync + type: string + required: false + default: "" + +env: + ANSIBLE_FORCE_COLOR: True + ANSIBLE_VAULT_PASSWORD_FILE: ${{ github.workspace }}/vault-pass + +jobs: + package-sync-version-test: + name: Sync specific package repository versions from Ark to Test Pulp + runs-on: [self-hosted, stackhpc-release-train] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Release Train & dependencies + uses: ./.github/actions/setup + with: + vault-password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + vault-password-file: ${{ env.ANSIBLE_VAULT_PASSWORD_FILE }} + + - name: Sync and publish package repositories in test + run: | + ansible-playbook -i ansible/inventory \ + ansible/test-pulp-repo-specific-version-query.yml \ + ansible/test-pulp-repo-sync.yml \ + ansible/test-pulp-repo-publication-cleanup.yml \ + ansible/test-pulp-repo-publish.yml \ + -e '{test_pulp_sync_repo_version: "$REPO_VERSION"}' + -e deb_package_repo_filter="'$FILTER'" \ + -e rpm_package_repo_filter="'$FILTER'" + env: + FILTER: ${{ github.event.inputs.filter }} + REPO_VERSION: ${{ github.event.inputs.repo_version }} diff --git a/ansible/inventory/group_vars/all/test-pulp-repos b/ansible/inventory/group_vars/all/test-pulp-repos index f3df4b3f..6c347aeb 100644 --- a/ansible/inventory/group_vars/all/test-pulp-repos +++ b/ansible/inventory/group_vars/all/test-pulp-repos @@ -24,7 +24,10 @@ test_pulp_repository_deb_repo_versions: {} test_pulp_repository_deb_repos: >- {%- set test_repos = [] -%} {%- for repo in deb_package_repos_filtered -%} - {%- if repo.sync | default(true) and repo.publish | default(true) -%} + {%- if repo.sync | default(true) and + repo.publish | default(true) and + (not test_pulp_sync_repo_version or + repo.short_name in test_pulp_repository_deb_repo_versions) -%} {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%} {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%} {%- if "distributions" in repo -%} @@ -51,7 +54,10 @@ test_pulp_distribution_deb_common: test_pulp_distribution_deb: >- {%- set test_dists = [] -%} {%- for repo in deb_package_repos_filtered -%} - {%- if repo.sync | default(true) and repo.publish | default(true) -%} + {%- if repo.sync | default(true) and + repo.publish | default(true) and + (not test_pulp_sync_repo_version or + repo.short_name in test_pulp_repository_deb_repo_versions) -%} {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%} {%- set name = repo.distribution_name ~ version ~ "-ark" -%} {%- set repo_name = repo.name ~ " (ark)" -%} @@ -88,7 +94,10 @@ test_pulp_repository_rpm_repo_versions: {} test_pulp_repository_rpm_repos: >- {%- set test_repos = [] -%} {%- for repo in rpm_package_repos_filtered -%} - {%- if repo.sync | default(true) and repo.publish | default(true) -%} + {%- if repo.sync | default(true) and + repo.publish | default(true) and + (not test_pulp_sync_repo_version or + repo.short_name in test_pulp_repository_rpm_repo_versions) -%} {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%} {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%} {%- set _ = test_repos.append(test_pulp_repository_rpm_repo_common | combine(test_repo)) -%} @@ -106,7 +115,10 @@ test_pulp_distribution_rpm_common: test_pulp_distribution_rpm: >- {%- set test_dists = [] -%} {%- for repo in rpm_package_repos_filtered -%} - {%- if repo.sync | default(true) and repo.publish | default(true) -%} + {%- if repo.sync | default(true) and + repo.publish | default(true) and + (not test_pulp_sync_repo_version or + repo.short_name in test_pulp_repository_rpm_repo_versions) -%} {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%} {%- set name = repo.distribution_name ~ version ~ "-ark" -%} {%- set repo_name = repo.name ~ " (ark)" -%} @@ -116,3 +128,6 @@ test_pulp_distribution_rpm: >- {%- endif -%} {%- endfor -%} {{ test_dists }} + +# A version string (%Y%m%dT%H%M%S) to sync to test pulp +test_pulp_sync_repo_version: "" diff --git a/ansible/test-pulp-repo-specific-version-query.yml b/ansible/test-pulp-repo-specific-version-query.yml new file mode 100644 index 00000000..edc4b7a5 --- /dev/null +++ b/ansible/test-pulp-repo-specific-version-query.yml @@ -0,0 +1,99 @@ +--- +# This playbook queries the dev Pulp server for distributions of each +# repository, which match a specific version string. +# The version is extracted from the base_path, and used to set the +# 'test_pulp_repository_deb_repo_versions' and +# 'test_pulp_repository_rpm_repo_versions' facts which contain repository +# versions to sync to the test Pulp server. + +- name: Update test repo versions + hosts: localhost + gather_facts: true + vars: + pulp_url: "{{ dev_pulp_url }}" + pulp_username: "{{ dev_pulp_username }}" + pulp_password: "{{ dev_pulp_password }}" + + tasks: + - name: Test repo version string formatting + ansible.builtin.debug: + msg: "{{ test_pulp_sync_repo_version | to_datetime('%Y%m%dT%H%M%S') }}" + when: test_pulp_sync_repo_version + register: test_formatting + ignore_errors: true + + - name: Fail when test_pulp_sync_repo_version is not of the correct format + ansible.builtin.fail: + msg: "test_pulp_sync_repo_version is not correctly formatted, use %Y%m%dT%H%M%S" + when: + - test_formatting.failed + + - name: Query Deb distributions + pulp.squeezer.deb_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: pulp_deb_dists_list + + - name: Query RPM distributions + pulp.squeezer.rpm_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: pulp_rpm_dists_list + + - name: Set a fact about deb versions to sync + ansible.builtin.set_fact: + test_pulp_repository_deb_repo_versions: >- + {%- set searchstring = item | dirname + '/' -%} + {%- + set shortname = deb_package_repos + | selectattr('base_path', 'defined') + | selectattr('base_path', 'equalto', searchstring) + | map(attribute='short_name') + -%} + {{ + ( + test_pulp_repository_deb_repo_versions | default({}) + | combine( + {shortname | first: item | basename} + ) + ) + if shortname | length > 0 + else ( + test_pulp_repository_deb_repo_versions | default({}) + ) + }} + loop: "{{ pulp_deb_dists_list.distributions | selectattr('base_path', 'search', test_pulp_sync_repo_version) | map(attribute='base_path') }}" + + - name: Display deb versions to sync fact + ansible.builtin.debug: + var: test_pulp_repository_deb_repo_versions + + - name: Set a fact about RPM versions to sync + ansible.builtin.set_fact: + test_pulp_repository_rpm_repo_versions: >- + {%- set searchstring = item | dirname + '/' -%} + {%- + set shortname = rpm_package_repos + | selectattr('base_path', 'defined') + | selectattr('base_path', 'equalto', searchstring) + | map(attribute='short_name') + -%} + {{ + ( + test_pulp_repository_rpm_repo_versions | default({}) + | combine( + {shortname | first: item | basename} + ) + ) + if shortname | length > 0 + else ( + test_pulp_repository_rpm_repo_versions | default({}) + ) + }} + loop: "{{ pulp_rpm_dists_list.distributions | selectattr('base_path', 'search', test_pulp_sync_repo_version) | map(attribute='base_path') }}" + + - name: Display RPM versions to sync fact + ansible.builtin.debug: + var: test_pulp_repository_rpm_repo_versions diff --git a/ansible/test-pulp-repo-sync.yml b/ansible/test-pulp-repo-sync.yml index 14b36666..e39a8d49 100644 --- a/ansible/test-pulp-repo-sync.yml +++ b/ansible/test-pulp-repo-sync.yml @@ -11,6 +11,7 @@ Some expected Deb repositories not present in sync versions: {{ missing_repos | join(',') }} vars: missing_repos: "{{ test_pulp_repository_deb_repos | map(attribute='short_name') | list | difference(test_pulp_repository_deb_repo_versions) | list }}" + when: not test_pulp_sync_repo_version - name: Assert that RPM versions variable is populated assert: @@ -20,6 +21,7 @@ Some expected RPM repositories not present in sync versions: {{ missing_repos | join(',') }} vars: missing_repos: "{{ test_pulp_repository_rpm_repos | map(attribute='short_name') | list | difference(test_pulp_repository_rpm_repo_versions) | list }}" + when: not test_pulp_sync_repo_version - import_role: name: stackhpc.pulp.pulp_repository