Merge pull request #238 from stackhpc/feat/sync-specific-repo-versions

Sync specific repo versions to Test Pulp from Ark

.github/workflows/package-sync-version-test-pulp.yml

@@ -0,0 +1,47 @@
+---
+name: Sync package repository versions to Test Pulp
+on:
+  workflow_dispatch:
+    inputs:
+      repo_version:
+        description: A repository version string maching repository versions to sync from Ark to Test Pulp
+        type: string
+        required: false
+        default: ""
+      filter:
+        description: Space-separated list of regular expressions matching short_name of repositories to sync
+        type: string
+        required: false
+        default: ""
+
+env:
+  ANSIBLE_FORCE_COLOR: True
+  ANSIBLE_VAULT_PASSWORD_FILE: ${{ github.workspace }}/vault-pass
+
+jobs:
+  package-sync-version-test:
+    name: Sync specific package repository versions from Ark to Test Pulp
+    runs-on: [self-hosted, stackhpc-release-train]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Setup Release Train & dependencies
+      uses: ./.github/actions/setup
+      with:
+        vault-password: ${{ secrets.ANSIBLE_VAULT_PASSWORD }}
+        vault-password-file: ${{ env.ANSIBLE_VAULT_PASSWORD_FILE }}
+
+    - name: Sync and publish package repositories in test
+      run: |
+        ansible-playbook -i ansible/inventory \
+        ansible/test-pulp-repo-specific-version-query.yml \
+        ansible/test-pulp-repo-sync.yml \
+        ansible/test-pulp-repo-publication-cleanup.yml \
+        ansible/test-pulp-repo-publish.yml \
+        -e '{test_pulp_sync_repo_version: "$REPO_VERSION"}'
+        -e deb_package_repo_filter="'$FILTER'" \
+        -e rpm_package_repo_filter="'$FILTER'"
+      env:
+        FILTER: ${{ github.event.inputs.filter }}
+        REPO_VERSION: ${{ github.event.inputs.repo_version }}

ansible/inventory/group_vars/all/test-pulp-repos

@@ -24,7 +24,10 @@ test_pulp_repository_deb_repo_versions: {}
 test_pulp_repository_deb_repos: >-
   {%- set test_repos = [] -%}
   {%- for repo in deb_package_repos_filtered -%}
-  {%- if repo.sync | default(true) and repo.publish | default(true) -%}
+  {%- if repo.sync | default(true) and
+        repo.publish | default(true) and
+          (not test_pulp_sync_repo_version or
+            repo.short_name in test_pulp_repository_deb_repo_versions) -%}
   {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%}
   {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%}
   {%- if "distributions" in repo -%}
@@ -51,7 +54,10 @@ test_pulp_distribution_deb_common:
 test_pulp_distribution_deb: >-
   {%- set test_dists = [] -%}
   {%- for repo in deb_package_repos_filtered -%}
-  {%- if repo.sync | default(true) and repo.publish | default(true) -%}
+  {%- if repo.sync | default(true) and
+        repo.publish | default(true) and
+          (not test_pulp_sync_repo_version or 
+            repo.short_name in test_pulp_repository_deb_repo_versions) -%}
   {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%}
   {%- set name = repo.distribution_name ~ version ~ "-ark" -%}
   {%- set repo_name = repo.name ~ " (ark)" -%}
@@ -88,7 +94,10 @@ test_pulp_repository_rpm_repo_versions: {}
 test_pulp_repository_rpm_repos: >-
   {%- set test_repos = [] -%}
   {%- for repo in rpm_package_repos_filtered -%}
-  {%- if repo.sync | default(true) and repo.publish | default(true) -%}
+  {%- if repo.sync | default(true) and
+        repo.publish | default(true) and 
+          (not test_pulp_sync_repo_version or
+            repo.short_name in test_pulp_repository_rpm_repo_versions) -%}
   {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%}
   {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%}
   {%- set _ = test_repos.append(test_pulp_repository_rpm_repo_common | combine(test_repo)) -%}
@@ -106,7 +115,10 @@ test_pulp_distribution_rpm_common:
 test_pulp_distribution_rpm: >-
   {%- set test_dists = [] -%}
   {%- for repo in rpm_package_repos_filtered -%}
-  {%- if repo.sync | default(true) and repo.publish | default(true) -%}
+  {%- if repo.sync | default(true) and
+        repo.publish | default(true) and
+          (not test_pulp_sync_repo_version or
+             repo.short_name in test_pulp_repository_rpm_repo_versions) -%}
   {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%}
   {%- set name = repo.distribution_name ~ version ~ "-ark" -%}
   {%- set repo_name = repo.name ~ " (ark)" -%}
@@ -116,3 +128,6 @@ test_pulp_distribution_rpm: >-
   {%- endif -%}
   {%- endfor -%}
   {{ test_dists }}
+
+# A version string (%Y%m%dT%H%M%S) to sync to test pulp
+test_pulp_sync_repo_version: ""

ansible/test-pulp-repo-specific-version-query.yml

@@ -0,0 +1,99 @@
+---
+# This playbook queries the dev Pulp server for distributions of each
+# repository, which match a specific version string.
+# The version is extracted from the base_path, and used to set the
+# 'test_pulp_repository_deb_repo_versions' and
+# 'test_pulp_repository_rpm_repo_versions' facts which contain repository
+# versions to sync to the test Pulp server.
+
+- name: Update test repo versions
+  hosts: localhost
+  gather_facts: true
+  vars:
+    pulp_url: "{{ dev_pulp_url }}"
+    pulp_username: "{{ dev_pulp_username }}"
+    pulp_password: "{{ dev_pulp_password }}"
+
+  tasks:
+    - name: Test repo version string formatting
+      ansible.builtin.debug:
+        msg: "{{ test_pulp_sync_repo_version | to_datetime('%Y%m%dT%H%M%S') }}"
+      when: test_pulp_sync_repo_version
+      register: test_formatting
+      ignore_errors: true
+
+    - name: Fail when test_pulp_sync_repo_version is not of the correct format
+      ansible.builtin.fail:
+        msg: "test_pulp_sync_repo_version is not correctly formatted, use %Y%m%dT%H%M%S"
+      when:
+        - test_formatting.failed
+
+    - name: Query Deb distributions
+      pulp.squeezer.deb_distribution:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+      register: pulp_deb_dists_list
+
+    - name: Query RPM distributions
+      pulp.squeezer.rpm_distribution:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+      register: pulp_rpm_dists_list
+
+    - name: Set a fact about deb versions to sync
+      ansible.builtin.set_fact:
+        test_pulp_repository_deb_repo_versions: >-
+          {%- set searchstring = item | dirname + '/' -%}
+          {%-
+            set shortname = deb_package_repos
+            | selectattr('base_path', 'defined')
+            | selectattr('base_path', 'equalto', searchstring)
+            | map(attribute='short_name')
+          -%}
+          {{
+            (
+              test_pulp_repository_deb_repo_versions | default({})
+              | combine(
+                {shortname | first: item | basename}
+              )
+            )
+            if shortname | length > 0
+            else (
+              test_pulp_repository_deb_repo_versions | default({})
+              )
+          }}
+      loop: "{{ pulp_deb_dists_list.distributions | selectattr('base_path', 'search', test_pulp_sync_repo_version) | map(attribute='base_path') }}"
+
+    - name: Display deb versions to sync fact
+      ansible.builtin.debug:
+        var: test_pulp_repository_deb_repo_versions
+
+    - name: Set a fact about RPM versions to sync
+      ansible.builtin.set_fact:
+        test_pulp_repository_rpm_repo_versions: >-
+          {%- set searchstring = item | dirname + '/' -%}
+          {%-
+            set shortname = rpm_package_repos
+            | selectattr('base_path', 'defined')
+            | selectattr('base_path', 'equalto', searchstring)
+            | map(attribute='short_name')
+          -%}
+          {{
+            (
+              test_pulp_repository_rpm_repo_versions | default({})
+              | combine(
+                {shortname | first: item | basename}
+              )
+            )
+            if shortname | length > 0
+            else (
+              test_pulp_repository_rpm_repo_versions | default({})
+              )
+          }}
+      loop: "{{ pulp_rpm_dists_list.distributions | selectattr('base_path', 'search', test_pulp_sync_repo_version) | map(attribute='base_path') }}"
+
+    - name: Display RPM versions to sync fact
+      ansible.builtin.debug:
+        var: test_pulp_repository_rpm_repo_versions

ansible/test-pulp-repo-sync.yml

@@ -11,6 +11,7 @@
           Some expected Deb repositories not present in sync versions: {{ missing_repos | join(',') }}
       vars:
         missing_repos: "{{ test_pulp_repository_deb_repos | map(attribute='short_name') | list | difference(test_pulp_repository_deb_repo_versions) | list }}"
+      when: not test_pulp_sync_repo_version
 
     - name: Assert that RPM versions variable is populated
       assert:
@@ -20,6 +21,7 @@
           Some expected RPM repositories not present in sync versions: {{ missing_repos | join(',') }}
       vars:
         missing_repos: "{{ test_pulp_repository_rpm_repos | map(attribute='short_name') | list | difference(test_pulp_repository_rpm_repo_versions) | list }}"
+      when: not test_pulp_sync_repo_version
 
     - import_role:
         name: stackhpc.pulp.pulp_repository