From c92795edb3e87b568152e832fe33d6a6d1a19d4d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 May 2024 09:29:49 +0100 Subject: [PATCH 1/6] CI: Add Grafana and OpenSearch Dashboards variables for SOT Depends-On: https://github.com/stackhpc/stackhpc-openstack-tests/pull/3 --- etc/kayobe/ansible/stackhpc-openstack-tests.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml index b99b9f91d..7775267c1 100644 --- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml +++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml @@ -65,18 +65,30 @@ --timeout {{ sot_timeout }} -vv environment: + GRAFANA_URL: "{{ sot_grafana_url }}" + GRAFANA_USERNAME: "{{ sot_grafana_username }}" + GRAFANA_PASSWORD: "{{ sot_grafana_password }}" OPENSEARCH_HOSTS: "{{ sot_opensearch_hosts }}" OPENSEARCH_PORT: "{{ sot_opensearch_port }}" OPENSEARCH_TLS: "{{ sot_opensearch_tls }}" + OPENSEARCH_DASHBOARDS_URL: "{{ sot_opensearch_dashboards_url }}" + OPENSEARCH_DASHBOARDS_USERNAME: "{{ sot_opensearch_dashboards_username }}" + OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sot_opensearch_dashboards_password }}" PROMETHEUS_URL: "{{ sot_prometheus_url }}" PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}" PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}" vars: kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}" kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}" + sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000" + sot_grafana_username: "grafana_local_admin" + sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}" sot_opensearch_hosts: "{{ kolla_internal_fqdn }}" sot_opensearch_port: 9200 sot_opensearch_tls: false + sot_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601" + sot_opensearch_dashboards_username: "opensearch" + sot_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}" sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091" sot_prometheus_username: "admin" sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}" From 7c21b46c3f8e0f9e9842c0144894518c2545cffc Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 May 2024 09:31:21 +0100 Subject: [PATCH 2/6] CI: Add Docker and SELinux variables for SOT Depends-On: https://github.com/stackhpc/stackhpc-openstack-tests/pull/4 --- etc/kayobe/ansible/stackhpc-openstack-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml index 7775267c1..7d6357162 100644 --- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml +++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml @@ -65,6 +65,8 @@ --timeout {{ sot_timeout }} -vv environment: + DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}" + DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}" GRAFANA_URL: "{{ sot_grafana_url }}" GRAFANA_USERNAME: "{{ sot_grafana_username }}" GRAFANA_PASSWORD: "{{ sot_grafana_password }}" @@ -77,9 +79,14 @@ PROMETHEUS_URL: "{{ sot_prometheus_url }}" PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}" PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}" + SELINUX_STATE: "{{ sot_selinux_state }}" vars: kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}" kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}" + # Inclusive min + sot_docker_version_min: "24.0.0" + # Exclusive max + sot_docker_version_max: "27.0.0" sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000" sot_grafana_username: "grafana_local_admin" sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}" @@ -92,6 +99,7 @@ sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091" sot_prometheus_username: "admin" sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}" + sot_selinux_state: "{{ selinux_state }}" always: - name: Fetch results ansible.builtin.fetch: From e1d1403c962114cd7932f3b18e1bc66cdc1bb544 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 May 2024 10:12:26 +0100 Subject: [PATCH 3/6] CI: Separate monitoring vs host test execution in SOT There are currently two types of tests in StackHPC OpenStack Tests: service-level tests such as those for monitoring, and host-level tests such as those for Docker and SELinux. We should execute the service-level tests from a single remote host, while the host-level tests should be executed on each of the overcloud hosts. This means that we now get several results files, so it becomes important to provide a summary of failures. --- .github/workflows/stackhpc-all-in-one.yml | 24 +++--- .../ansible/stackhpc-openstack-tests.yml | 85 +++++++++++++++---- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 611a56699..d25a441da 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -383,8 +383,6 @@ jobs: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} - name: StackHPC OpenStack tests - id: stackhpc-openstack-tests - continue-on-error: true run: | mkdir -p sot-results docker run -t --rm \ @@ -420,16 +418,20 @@ jobs: sot-results/ if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }} - - name: Fail if any Tempest tests failed + - name: Fail if any tests failed run: | - test $(wc -l < tempest-artifacts/failed-tests) -lt 1 - - - name: Fail if any StackHPC OpenStack tests failed - run: | - echo "Some StackHPC OpenStack tests failed." - echo "See HTML results artifact (sot-results) for details." - exit 1 - if: steps.stackhpc-openstack-tests.outcome == 'failure' + rc=0 + if [[ $(wc -l < tempest-artifacts/failed-tests) -ne 0 ]]; then + echo "Some Tempest tests failed." + echo "See HTML results artifact (tempest-artifacts) for details." + rc=1 + fi + if [[ $(wc -l < sot-results/failed-tests) -ne 0 ]]; then + echo "Some StackHPC OpenStack tests failed." + echo "See HTML results artifact (sot-results) for details." + rc=1 + fi + exit $rc - name: Destroy run: terraform destroy -auto-approve diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml index 7d6357162..3ed188f10 100644 --- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml +++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml @@ -1,6 +1,9 @@ --- +# This playbook executes tests from the StackHPC OpenStack Tests repository. +# https://github.com/stackhpc/stackhpc-openstack-tests + - name: Run StackHPC OpenStack tests - hosts: tempest_runner + hosts: tempest_runner:overcloud tags: - stackhpc-openstack-tests vars: @@ -10,6 +13,11 @@ sot_timeout: 30 results_path_local: "{{ lookup('env', 'HOME') }}/sot-results" tasks: + - name: Assert that there is only one host in the tempest_runner group + assert: + that: groups.get('tempest_runner', []) | length == 1 + fail_msg: The tempest_runner group should contain exactly one host + - block: - name: Create a temporary directory for tests repo ansible.builtin.tempfile: @@ -55,18 +63,18 @@ file: "{{ kayobe_env_config_path }}/kolla/passwords.yml" name: kolla_passwords - - name: Run StackHPC OpenStack tests + # Monitoring tests should run once, executed on the host in the + # tempest_runner group. + - name: Run StackHPC OpenStack monitoring tests ansible.builtin.command: cmd: > {{ sot_venv }}/bin/py.test - --html={{ results_tmpdir.path }}/stackhpc-openstack-tests.html + --html={{ results_tmpdir.path }}/monitoring.html --self-contained-html - --pyargs stackhpc_openstack_tests + --pyargs stackhpc_openstack_tests.monitoring --timeout {{ sot_timeout }} -vv environment: - DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}" - DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}" GRAFANA_URL: "{{ sot_grafana_url }}" GRAFANA_USERNAME: "{{ sot_grafana_username }}" GRAFANA_PASSWORD: "{{ sot_grafana_password }}" @@ -79,14 +87,9 @@ PROMETHEUS_URL: "{{ sot_prometheus_url }}" PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}" PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}" - SELINUX_STATE: "{{ sot_selinux_state }}" vars: kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}" kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}" - # Inclusive min - sot_docker_version_min: "24.0.0" - # Exclusive max - sot_docker_version_max: "27.0.0" sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000" sot_grafana_username: "grafana_local_admin" sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}" @@ -99,13 +102,65 @@ sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091" sot_prometheus_username: "admin" sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}" + failed_when: monitoring_results.rc not in [0, 1] + register: monitoring_results + when: "'tempest_runner' in group_names" + + # Host tests should run on every host in the overcloud group. + # TODO: Use TestInfra's native Ansible or SSH connection plugins for + # remote test execution? That would place all results in a single file + # and allow us to execute all tests from a single host. + # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends + - name: Run StackHPC OpenStack host tests + ansible.builtin.command: + cmd: > + {{ sot_venv }}/bin/py.test + --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html + --self-contained-html + --pyargs stackhpc_openstack_tests.host + --timeout {{ sot_timeout }} + -vv + environment: + DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}" + DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}" + SELINUX_STATE: "{{ sot_selinux_state }}" + vars: + # Inclusive min + sot_docker_version_min: "24.0.0" + # Exclusive max + sot_docker_version_max: "27.0.0" sot_selinux_state: "{{ selinux_state }}" + failed_when: host_results.rc not in [0, 1] + register: host_results + when: "'overcloud' in group_names" always: - - name: Fetch results - ansible.builtin.fetch: - src: "{{ results_tmpdir.path }}/stackhpc-openstack-tests.html" + - name: Synchronize results + ansible.posix.synchronize: + src: "{{ results_tmpdir.path }}/" dest: "{{ results_path_local }}/" - flat: true + mode: pull + archive: no + recursive: true + # For jump host + use_ssh_args: true + + - name: Write a file containing failed test runs + ansible.builtin.copy: + content: |- + {% for host in ansible_play_hosts_all %} + {% if host not in ansible_play_hosts %} + {{ host }}: Host failure + {% endif %} + {% if hostvars[host].monitoring_results.rc | default(0) != 0 %} + monitoring.html + {% endif %} + {% if hostvars[host].host_results.rc | default(0) != 0 %} + host-{{ host }}.html + {% endif %} + {% endfor %} + dest: "{{ results_path_local }}/failed-tests" + delegate_to: localhost + run_once: true - name: Clean up temporary directory ansible.builtin.file: From 2d2c0013ef6d1a8c79c9487b33b18f400a1bf044 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 20 May 2024 14:44:23 +0000 Subject: [PATCH 4/6] SOT: Support OpenSearch and Dashboards TLS --- etc/kayobe/ansible/stackhpc-openstack-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml index 3ed188f10..9c7238058 100644 --- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml +++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml @@ -84,6 +84,7 @@ OPENSEARCH_DASHBOARDS_URL: "{{ sot_opensearch_dashboards_url }}" OPENSEARCH_DASHBOARDS_USERNAME: "{{ sot_opensearch_dashboards_username }}" OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sot_opensearch_dashboards_password }}" + OPENSEARCH_DASHBOARDS_CACERT: "{{ sot_opensearch_dashboards_cacert }}" PROMETHEUS_URL: "{{ sot_prometheus_url }}" PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}" PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}" @@ -95,10 +96,11 @@ sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}" sot_opensearch_hosts: "{{ kolla_internal_fqdn }}" sot_opensearch_port: 9200 - sot_opensearch_tls: false + sot_opensearch_tls: "{{ kolla_enable_tls_internal | bool }}" sot_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601" sot_opensearch_dashboards_username: "opensearch" sot_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}" + sot_opensearch_dashboards_cacert: "TODO" sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091" sot_prometheus_username: "admin" sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}" From dc053e1c4ed23310a660bfad09f9c703b37700b3 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 20 May 2024 14:45:17 +0000 Subject: [PATCH 5/6] ci-multinode: Add internal interface to seed This is required for StackHPC OpenStack tests to access OpenSearch and Prometheus. --- .../inventory/group_vars/seed/network-interfaces | 2 ++ etc/kayobe/environments/ci-multinode/seed.yml | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces index e604b02ec..03aca06ec 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces @@ -4,6 +4,8 @@ admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + provision_oc_interface: "{{ vxlan_interfaces[0].device}}.{{ provision_oc_vlan }}" external_interface: "{{ vxlan_interfaces[0].device }}.{{ external_vlan }}" diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml index bb9e3c6bf..b177f7fe3 100644 --- a/etc/kayobe/environments/ci-multinode/seed.yml +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -3,7 +3,7 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl seed_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" -seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}" +seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + seed_extra_network_interfaces_internal + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}" # Seed has been provided an external interface # for tempest tests and SSH access to machines. @@ -11,6 +11,10 @@ seed_extra_network_interfaces_external: - "external" - "public" +# Seed has been provided an internal API interface for StackHPC OpenStack tests. +seed_extra_network_interfaces_internal: + - "internal" + # Seed requires access to the storage network for manila-cephfs. seed_extra_network_interfaces_manila: - "storage" From c76545500076ee77a7a17effefaf3a675e8e0357 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 13 Jan 2025 13:05:07 +0000 Subject: [PATCH 6/6] dnm: test new branch pin --- etc/kayobe/ansible/stackhpc-openstack-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml index 9c7238058..7c6433c22 100644 --- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml +++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml @@ -9,7 +9,8 @@ vars: sot_venv: "{{ virtualenv_path }}/sot-venv" sot_repo: "https://github.com/stackhpc/stackhpc-openstack-tests" - sot_version: "v0.0.1" + # FIXME + sot_version: "docker-selinux" sot_timeout: 30 results_path_local: "{{ lookup('env', 'HOME') }}/sot-results" tasks: