More StackHPC OpenStack tests

.github/workflows/stackhpc-all-in-one.yml

@@ -383,8 +383,6 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
       - name: StackHPC OpenStack tests
-        id: stackhpc-openstack-tests
-        continue-on-error: true
         run: |
           mkdir -p sot-results
           docker run -t --rm \
@@ -420,16 +418,20 @@ jobs:
             sot-results/
         if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
 
-      - name: Fail if any Tempest tests failed
+      - name: Fail if any tests failed
         run: |
-          test $(wc -l < tempest-artifacts/failed-tests) -lt 1
-
-      - name: Fail if any StackHPC OpenStack tests failed
-        run: |
-          echo "Some StackHPC OpenStack tests failed."
-          echo "See HTML results artifact (sot-results) for details."
-          exit 1
-        if: steps.stackhpc-openstack-tests.outcome == 'failure'
+          rc=0
+          if [[ $(wc -l < tempest-artifacts/failed-tests) -ne 0 ]]; then
+            echo "Some Tempest tests failed."
+            echo "See HTML results artifact (tempest-artifacts) for details."
+            rc=1
+          fi
+          if [[ $(wc -l < sot-results/failed-tests) -ne 0 ]]; then
+            echo "Some StackHPC OpenStack tests failed."
+            echo "See HTML results artifact (sot-results) for details."
+            rc=1
+          fi
+          exit $rc
 
       - name: Destroy
         run: terraform destroy -auto-approve

.github/workflows/stackhpc-pull-request.yml

@@ -7,6 +7,8 @@ concurrency:
 name: Pull request
 'on':
   pull_request:
+  # FIXME: Remove
+  push:
 jobs:
   # Detect which files have changed and use this to run jobs conditionally.
   # Note that we can't use the workflow-level paths attribute since this
@@ -106,92 +108,3 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-ubuntu-jammy-ovn:
-    name: aio (Ubuntu Jammy OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: ubuntu
-      os_release: jammy
-      ssh_username: ubuntu
-      neutron_plugin: ovn
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-rocky-9-ovs:
-    name: aio (Rocky 9 OVS)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovs
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-rocky-9-ovn:
-    name: aio (Rocky 9 OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovn
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  # Test two upgrade scenarios: Ubuntu Jammy OVS and Rocky 9 OVN.
-
-  all-in-one-upgrade-ubuntu-jammy-ovs:
-    name: aio upgrade (Ubuntu Jammy OVS)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: ubuntu
-      os_release: jammy
-      ssh_username: ubuntu
-      neutron_plugin: ovs
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-      upgrade: true
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-upgrade-rocky-9-ovn:
-    name: aio upgrade (Rocky 9 OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovn
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-      upgrade: true
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

etc/kayobe/ansible/stackhpc-openstack-tests.yml

@@ -1,15 +1,24 @@
 ---
+# This playbook executes tests from the StackHPC OpenStack Tests repository.
+# https://github.com/stackhpc/stackhpc-openstack-tests
+
 - name: Run StackHPC OpenStack tests
-  hosts: tempest_runner
+  hosts: tempest_runner:overcloud
   tags:
     - stackhpc-openstack-tests
   vars:
     sot_venv: "{{ virtualenv_path }}/sot-venv"
     sot_repo: "https://github.com/stackhpc/stackhpc-openstack-tests"
-    sot_version: "v0.0.1"
+    # FIXME
+    sot_version: "docker-selinux"
     sot_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sot-results"
   tasks:
+    - name: Assert that there is only one host in the tempest_runner group
+      assert:
+        that: groups.get('tempest_runner', []) | length == 1
+        fail_msg: The tempest_runner group should contain exactly one host
+
     - block:
         - name: Create a temporary directory for tests repo
           ansible.builtin.tempfile:
@@ -55,37 +64,106 @@
             file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
             name: kolla_passwords
 
-        - name: Run StackHPC OpenStack tests
+        # Monitoring tests should run once, executed on the host in the
+        # tempest_runner group.
+        - name: Run StackHPC OpenStack monitoring tests
           ansible.builtin.command:
             cmd: >
               {{ sot_venv }}/bin/py.test
-              --html={{ results_tmpdir.path }}/stackhpc-openstack-tests.html
+              --html={{ results_tmpdir.path }}/monitoring.html
               --self-contained-html
-              --pyargs stackhpc_openstack_tests
+              --pyargs stackhpc_openstack_tests.monitoring
               --timeout {{ sot_timeout }}
               -vv
           environment:
+            GRAFANA_URL: "{{ sot_grafana_url }}"
+            GRAFANA_USERNAME: "{{ sot_grafana_username }}"
+            GRAFANA_PASSWORD: "{{ sot_grafana_password }}"
             OPENSEARCH_HOSTS: "{{ sot_opensearch_hosts }}"
             OPENSEARCH_PORT: "{{ sot_opensearch_port }}"
             OPENSEARCH_TLS: "{{ sot_opensearch_tls }}"
+            OPENSEARCH_DASHBOARDS_URL: "{{ sot_opensearch_dashboards_url }}"
+            OPENSEARCH_DASHBOARDS_USERNAME: "{{ sot_opensearch_dashboards_username }}"
+            OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sot_opensearch_dashboards_password }}"
+            OPENSEARCH_DASHBOARDS_CACERT: "{{ sot_opensearch_dashboards_cacert }}"
             PROMETHEUS_URL: "{{ sot_prometheus_url }}"
             PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}"
             PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}"
           vars:
             kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
             kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
+            sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
+            sot_grafana_username: "grafana_local_admin"
+            sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
             sot_opensearch_hosts: "{{ kolla_internal_fqdn }}"
             sot_opensearch_port: 9200
-            sot_opensearch_tls: false
+            sot_opensearch_tls: "{{ kolla_enable_tls_internal | bool }}"
+            sot_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
+            sot_opensearch_dashboards_username: "opensearch"
+            sot_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
+            sot_opensearch_dashboards_cacert: "TODO"
             sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
             sot_prometheus_username: "admin"
             sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
+          failed_when: monitoring_results.rc not in [0, 1]
+          register: monitoring_results
+          when: "'tempest_runner' in group_names"
+
+        # Host tests should run on every host in the overcloud group.
+        # TODO: Use TestInfra's native Ansible or SSH connection plugins for
+        # remote test execution? That would place all results in a single file
+        # and allow us to execute all tests from a single host.
+        # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
+        - name: Run StackHPC OpenStack host tests
+          ansible.builtin.command:
+            cmd: >
+              {{ sot_venv }}/bin/py.test
+              --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
+              --self-contained-html
+              --pyargs stackhpc_openstack_tests.host
+              --timeout {{ sot_timeout }}
+              -vv
+          environment:
+            DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}"
+            DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}"
+            SELINUX_STATE: "{{ sot_selinux_state }}"
+          vars:
+            # Inclusive min
+            sot_docker_version_min: "24.0.0"
+            # Exclusive max
+            sot_docker_version_max: "27.0.0"
+            sot_selinux_state: "{{ selinux_state }}"
+          failed_when: host_results.rc not in [0, 1]
+          register: host_results
+          when: "'overcloud' in group_names"
       always:
-        - name: Fetch results
-          ansible.builtin.fetch:
-            src: "{{ results_tmpdir.path }}/stackhpc-openstack-tests.html"
+        - name: Synchronize results
+          ansible.posix.synchronize:
+            src: "{{ results_tmpdir.path }}/"
             dest: "{{ results_path_local }}/"
-            flat: true
+            mode: pull
+            archive: no
+            recursive: true
+            # For jump host
+            use_ssh_args: true
+
+        - name: Write a file containing failed test runs
+          ansible.builtin.copy:
+            content: |-
+              {% for host in ansible_play_hosts_all %}
+              {% if host not in ansible_play_hosts %}
+              {{ host }}: Host failure
+              {% endif %}
+              {% if hostvars[host].monitoring_results.rc | default(0) != 0 %}
+              monitoring.html
+              {% endif %}
+              {% if hostvars[host].host_results.rc | default(0) != 0 %}
+              host-{{ host }}.html
+              {% endif %}
+              {% endfor %}
+            dest: "{{ results_path_local }}/failed-tests"
+          delegate_to: localhost
+          run_once: true
 
         - name: Clean up temporary directory
           ansible.builtin.file:

etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces

@@ -4,6 +4,8 @@
 
 admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}"
 
+internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}"
+
 provision_oc_interface: "{{ vxlan_interfaces[0].device}}.{{ provision_oc_vlan }}"
 
 external_interface: "{{ vxlan_interfaces[0].device }}.{{ external_vlan }}"

etc/kayobe/environments/ci-multinode/seed.yml

@@ -3,14 +3,18 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl
 seed_lvm_groups:
   - "{{ stackhpc_lvm_group_rootvg }}"
 
-seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}"
+seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + seed_extra_network_interfaces_internal + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}"
 
 # Seed has been provided an external interface
 # for tempest tests and SSH access to machines.
 seed_extra_network_interfaces_external:
   - "external"
   - "public"
 
+# Seed has been provided an internal API interface for StackHPC OpenStack tests.
+seed_extra_network_interfaces_internal:
+  - "internal"
+
 # Seed requires access to the storage network for manila-cephfs.
 seed_extra_network_interfaces_manila:
   - "storage"