playbooks for CI checks, units and functional tests (#2079)

Playbooks to emulate the plugin template's testing paths and give direct control over the python version under test.

No-Issue

Signed-off-by: James Tanner <[email protected]>

.coveragerc

@@ -0,0 +1,5 @@
+[run]
+data_file = /tmp/.coverage
+omit = 
+ galaxy_ng/_vendor/*
+ galaxy_ng/tests/*

.github/workflows/ci_full.yml

@@ -0,0 +1,101 @@
+---
+name: galaxy_ng/ci
+on: {pull_request: {branches: ['**']}, push: {branches: ['**']}}
+
+jobs:
+
+ check_commit:
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Checkout code
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.after }} # for PR avoids checking out merge commit
+ fetch-depth: 0 # include all history
+
+ - name: Run script to validate commits for both pull request and a push
+ env:
+ PY_COLORS: '1'
+ ANSIBLE_FORCE_COLOR: '1'
+ GITHUB_PULL_REQUEST: ${{ github.event.number }}
+ GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
+ GITHUB_BRANCH: ${{ github.head_ref }}
+ GITHUB_REPO_SLUG: ${{ github.repository }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+ GITHUB_USER: ${{ github.event.pull_request.user.login }}
+ GITHUB_PR_COMMITS_URL: ${{ github.event.pull_request.commits_url }}
+ START_COMMIT: ${{ github.event.before }}
+ END_COMMIT: ${{ github.event.after }}
+ run: |
+ python .ci/scripts/validate_commit_message_custom.py
+
+ lint_po:
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Checkout code
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.after }} # for PR avoids checking out merge commit
+ fetch-depth: 0 # include all history
+
+ - name: intsall and run lint-po
+ run: |
+ pip install lint-po
+ lint-po ./galaxy_ng/locale/*/LC_MESSAGES/*.po
+
+ lint:
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Checkout code
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.after }} # for PR avoids checking out merge commit
+ fetch-depth: 0 # include all history
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+
+ - name: Install requirements
+ run: pip3 install -r lint_requirements.txt
+
+ - name: Run extra lint checks
+ run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh"
+
+ - name: Check manifest
+ run: check-manifest
+
+ - name: Check for pulpcore imports outside of pulpcore.plugin
+ run: sh .ci/scripts/check_pulpcore_imports.sh
+
+ - name: Check for gettext problems
+ run: sh .ci/scripts/check_gettext.sh
+
+ test:
+ runs-on: ubuntu-latest
+ steps:
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+
+ - uses: actions/checkout@v4
+
+ - name: install ansible
+ run: pip3 install ansible
+
+ - name: run the build container playbook
+ run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v build_container.yaml
+
+ - name: run the start container playbook
+ run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v start_container.yaml
+
+ - name: run the unit test playbook
+ run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v run_unit_tests.yaml
+
+ - name: run the functional test playbook
+ run: cd dev/playbooks; ansible-playbook -i 'localhost,' --forks=1 -v run_functional_tests.yaml

MANIFEST.in

@@ -22,3 +22,4 @@ recursive-include galaxy_ng *.py
 include galaxy_ng/tests/integration/utils/gpg/collection_sign.sh
 include galaxy_ng/tests/integration/utils/gpg/qe-sign-priv.gpg
 include galaxy-operator/bin/readyz.py
+exclude .coveragerc

dev/playbooks/RUNALL.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+docker-killall
+docker-rmall
+
+ANSIBLE="ansible-playbook -i 'localhost,' --forks=1 -vvvv"
+
+$ANSIBLE build_container.yaml
+$ANSIBLE start_container.yaml
+$ANSIBLE run_unit_tests.yaml

dev/playbooks/build_container.yaml

@@ -0,0 +1,34 @@
+# Ansible playbook to create the pulp service containers image
+---
+- hosts: localhost
+ connection: local
+ gather_facts: false
+ vars_files:
+ - vars/main.yaml
+
+ tasks:
+
+ - name: "Build the base image"
+ galaxy_ng.tools.local_run:
+ command: "cd docker; ./BUILD.sh"
+
+ - name: "Clean out the cache directory"
+ shell: "sudo rm -rf cache"
+
+ - name: "Make the cache directory"
+ file:
+ name: cache
+ state: directory
+
+ - name: "Generate Containerfile from template"
+ template:
+ src: Containerfile.j2
+ dest: cache/Containerfile
+
+ - name: "Build pulp image"
+ galaxy_ng.tools.local_run:
+ command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}//cache/Containerfile ../../.."
+
+ - name: "Clean image cache"
+ docker_prune:
+ images : true

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/action/local_run.py

@@ -0,0 +1,23 @@
+from ansible.plugins.action import ActionBase
+import subprocess
+
+
+class ActionModule(ActionBase):
+ TRANSFERS_FILES = False
+ _VALID_ARGS = frozenset(('command',))
+
+ def run(self, tmp=None, task_vars=None):
+ super(ActionModule, self).run(tmp, task_vars)
+
+ # Retrieve the command from the task's arguments
+ command = self._task.args.get('command', None)
+
+ if command is None:
+ return {"failed": True, "msg": "The 'command' argument is required"}
+
+ try:
+ # Run the command without capturing stdout or stderr
+ subprocess.run(command, shell=True, check=True)
+ return {"changed": True, "msg": "Command executed successfully"}
+ except subprocess.CalledProcessError as e:
+ return {"failed": True, "msg": "Command execution failed", "error": str(e)}

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/plugins/filter/repr.py

@@ -0,0 +1,31 @@
+from __future__ import absolute_import, division, print_function
+from packaging.version import parse as parse_version
+
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {
+ "metadata_version": "1.1",
+ "status": ["preview"],
+ "supported_by": "community",
+}
+
+
+def _repr_filter(value):
+ return repr(value)
+
+
+def _canonical_semver_filter(value):
+ return str(parse_version(value))
+
+
+# ---- Ansible filters ----
+class FilterModule(object):
+ """Repr filter."""
+
+ def filters(self):
+ """Filter associations."""
+ return {
+ "repr": _repr_filter,
+ "canonical_semver": _canonical_semver_filter,
+ }

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/roles/pulp_client/tasks/main.yml

@@ -0,0 +1,33 @@
+# https://github.com/pulp/oci_env/blob/main/base/local_scripts/generate_client.sh
+# https://github.com/pulp/oci_env/blob/main/base/container_scripts/install_client.sh
+
+- name: make sure there's a checkout of pulp-openapi-generator
+ shell: test -d ../../../pulp-openapi-generator || git clone https://github.com/pulp/pulp-openapi-generator ../../../pulp-openapi-generator
+ connection: local
+
+- name: get the pulp container IP
+ shell: docker inspect pulp | jq '.[0].NetworkSettings.Networks.pulp_ci_bridge.IPAddress' | tr -d '"'
+ connection: local
+ register: pulp_ip
+
+- name: run the generate script
+ galaxy_ng.tools.local_run:
+ command: >
+ cd ../../../pulp-openapi-generator;
+ export USE_LOCAL_API_JSON=true;
+ curl -L -k -u admin:password -o api.json 'https://{{ pulp_ip.stdout }}/api/galaxy/pulp/api/v3/docs/api.json?bindings&plugin={{ item }}';
+ bash -x generate.sh {{ item }} python
+ connection: local
+ loop:
+ - galaxy_ng
+ - pulp_ansible
+ - pulp_container
+ - pulpcore
+
+- name: install the generated client inside the pulp container
+ shell: cd /src/pulp-openapi-generator/{{ item }}-client/; pip3 install -e .
+ loop:
+ - galaxy_ng
+ - pulp_ansible
+ - pulp_container
+ - pulpcore

dev/playbooks/collections/ansible_collections/galaxy_ng/tools/roles/pulp_smash/tasks/main.yml

@@ -0,0 +1,79 @@
+- name: "Make /opt/oci_env/base/container_scripts"
+ file:
+ name: /opt/oci_env/base/container_scripts
+ state: directory
+
+- name: "Get the dynaconf script"
+ command:
+ cmd: "curl -o /opt/oci_env/base/container_scripts/get_dynaconf_var.sh https://raw.githubusercontent.com/pulp/oci_env/main/base/container_scripts/get_dynaconf_var.sh"
+
+#- name: install the pulpcli
+# command:
+# cmd: "pip3 install pulp-cli"
+
+#- name: "run pulp config"
+# shell: |
+# #/bin/bash
+# PULP_API_ROOT="/api/galaxy/pulp/"
+# pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT"
+
+- name: "Install pulp-smash"
+ command:
+ #cmd: "pip3 show pulp-smash || pip3 install git+https://github.com/pulp/pulp-smash.git"
+ cmd: "pip3 install git+https://github.com/pulp/pulp-smash.git"
+
+- name: "Get the pulp-smash setup script"
+ command:
+ cmd: "curl -o /tmp/configure_pulp_smash.sh https://raw.githubusercontent.com/pulp/oci_env/main/base/container_scripts/configure_pulp_smash.sh"
+
+- name: "Run the pulp smash config"
+ command:
+ cmd: "bash /tmp/configure_pulp_smash.sh"
+ environment:
+ API_HOST: "pulp"
+ API_PORT: "443"
+ API_PROTOCOL: "https"
+ DJANGO_SUPERUSER_USERNAME: "admin"
+ DJANGO_SUPERUSER_PASSWORD: "password"
+
+- name: "Make sure the smash config was made correctly"
+ command: "jq . /opt/settings/pulp_smash/settings.json"
+
+- name: "Set perms on the pulp_smash directory"
+ file:
+ path: ~/.config/pulp_smash/
+ state: directory
+ mode: "0777"
+
+- name: "Copy settings to config dir"
+ shell: "cp -f /opt/settings/pulp_smash/settings.json ~/.config/pulp_smash/setting.json"
+
+- name: "Set perms on the pulp_smash settings file"
+ file:
+ path: ~/.config/pulp_smash/setting.json
+ state: file
+ mode: "0666"
+
+- name: "Setup the ca-certs directory"
+ file:
+ path: /usr/local/share/ca-certificates/
+ state: directory
+
+- name: "Copy the cert to the ca folder"
+ shell: cp -f /etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/.
+
+#- name: "Add the CA cert"
+# script:
+# interpreter: /bin/bash
+# content: |
+# #!/bin/bash
+# # Hack: adding pulp CA to certifi.where()
+# CERTIFI=$(python3 -c 'import certifi; print(certifi.where())')
+# cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null
+# if [[ "$TEST" = "azure" ]]; then
+# cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null
+# fi
+# # Hack: adding pulp CA to default CA file
+# CERT=$(python3 -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)')
+# cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null
+# # update-ca-certificates

dev/playbooks/docker/BUILD.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ ! -d pulp-oci-images ]]; then
+ git clone https://github.com/pulp/pulp-oci-images
+fi
+cd pulp-oci-images
+git reset --hard
+cd ..
+
+cp -f switch_python pulp-oci-images/images/assets/.
+chmod +x pulp-oci-images/images/assets/switch_python
+
+cd pulp-oci-images
+git apply ../py311.patch
+
+docker build --file images/Containerfile.core.base --tag pulp/base:latest .
+docker build --file images/pulp_ci_centos/Containerfile --tag pulp/pulp-ci-centos9:latest .
+

dev/playbooks/docker/py311.patch

@@ -0,0 +1,15 @@
+diff --git a/images/Containerfile.core.base b/images/Containerfile.core.base
+index c05669c..7d84cb8 100644
+--- a/images/Containerfile.core.base
++++ b/images/Containerfile.core.base
+@@ -130,6 +130,10 @@ COPY images/assets/add_signing_service.sh /usr/bin/add_signing_service.sh
+ COPY images/assets/pulp-api /usr/bin/pulp-api
+ COPY images/assets/pulp-content /usr/bin/pulp-content
+ COPY images/assets/pulp-worker /usr/bin/pulp-worker
++COPY images/assets/switch_python /usr/bin/switch_python
++
++ARG PYTHON_VERSION=3.11
++RUN switch_python "$PYTHON_VERSION"
+
+ # Need to precreate when running pulp as the pulp user
+ RUN touch /var/log/galaxy_api_access.log && \

dev/playbooks/docker/switch_python

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+NEWV=$1
+if [[ "$NEWV" == "3.8" ]]; then
+ echo "using default python 3.8"
+ exit 0
+fi
+
+echo "switching python to $NEWV"
+
+rm -f /usr/local/bin/pip3
+dnf install -y python${NEWV} python${NEWV}-pip python${NEWV}-devel
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${NEWV} 1
+update-alternatives --set python3 /usr/bin/python${NEWV}
+update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip-${NEWV} 1
+update-alternatives --set pip3 /usr/bin/pip-${NEWV}