capi-image-templates

review changes

default to ovn

dns

unnecessary fields

label format change

dashes to underscores

unnecessary

underscores

update requirements for new coe template ansible role

default dns

underscores

init version of automated template generation

init version of automated template generation

init version of automated template generation

make operation no-op when no new images are defined and stop new images being included in old image list

list concat for images

etc/openstack-config/openstack-config.yml

@@ -39,8 +39,10 @@
 # Configuration of Glance software images.
 
 # List of Glance images. Format is as required by the stackhpc.os-images role.
-#openstack_images:
+#glance_images:
 
+# Images to be uploaded 
+#openstack_images: "{{ glance_images + kubernetes_images }}"
 # List of Diskimage Builder (DIB) elements paths to include in image builds.
 #openstack_image_elements:
 
@@ -57,4 +59,4 @@
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
-workaround_ansible_issue_8743: yes
+workaround_ansible_issue_8743: yes

examples/capi-templates-images.yml

@@ -0,0 +1,117 @@
+###############################################################################
+# Configuration of Glance software images.
+
+# Flavor must have a minimum of 2 VCPUs
+magnum_flavor: "m1.small"
+
+# Network to create tenant cluster FIPs on 
+magnum_external_network: "external"
+
+# Provider for cluster loadbalancers
+magnum_octavia_provider: "ovn"
+
+# helm chart version to use for tenant clusters
+magnum_helm_chart_version: "openstack-cluster-0.1.1-dev.0.main.221"
+
+ubuntu-focal-kube-v1_25_11:
+  name: "ubuntu-focal-kube-v1.25.11"
+  type: qcow2
+  image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.25.11-230712-0939.qcow2"
+  is_public: True
+  properties:
+    os_distro: "ubuntu"
+    os_version: "20.04"
+    kube_version: "v1.25.11"
+
+ubuntu-focal-kube-v1_26_6:
+  name: "ubuntu-focal-kube-v1.26.6"
+  type: qcow2
+  image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.26.6-230712-1010.qcow2"
+  is_public: True
+  properties:
+    os_distro: "ubuntu"
+    os_version: "20.04"
+    kube_version: "v1.26.6"
+
+ubuntu-focal-kube-v1_27_3:
+  name: "ubuntu-focal-kube-v1.27.3"
+  type: qcow2
+  image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.27.3-230712-1021.qcow2"
+  is_public: True
+  properties:
+    os_distro: "ubuntu"
+    os_version: "20.04"
+    kube_version: "v1.27.3"
+
+# List of Glance images. Format is as required by the stackhpc.os-images role.
+openstack_images:
+  - "{{ ubuntu-focal-kube-v1_25_11 }}"
+  - "{{ ubuntu-focal-kube-v1_26_6 }}"
+  - "{{ ubuntu-focal-kube-v1_27_3 }}"
+
+###############################################################################
+# Configuration of Magnum container clusters.
+
+kube_v1_25_11:
+  labels:
+    monitoring_enabled: "false"
+    kube_dashboard_enabled: "true"
+    capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
+    octavia_provider: "{{ magnum_octavia_provider }}"
+  external_network_id: "{{ magnum_external_network }}"
+  master_flavor: "{{ magnum_flavor }}"
+  flavor: "{{ magnum_flavor }}"
+  image: "ubuntu-focal-kube-v1.25.11"
+  name: "kubernetes-v1.25.11"
+  coe: "kubernetes"
+  network_driver: "calico"
+  master_lb_enabled: True
+  floating_ip_enabled: True
+  # Magnum’s default value for dns_nameserver is 8.8.8.8.
+  dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
+  public: True
+
+kube_v1_26_6:
+  labels:
+    monitoring_enabled: "false"
+    kube_dashboard_enabled: "true"
+    capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
+    octavia_provider: "{{ magnum_octavia_provider }}"
+  external_network_id: "{{ magnum_external_network }}"
+  master_flavor: "{{ magnum_flavor }}"
+  flavor: "{{ magnum_flavor }}"
+  image: "ubuntu-focal-kube-v1.26.6"
+  name: "kubernetes-v1.26.6"
+  coe: "kubernetes"
+  network_driver: "calico"
+  master_lb_enabled: True
+  floating_ip_enabled: True
+  # Magnum’s default value for dns_nameserver is 8.8.8.8.
+  dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
+  public: True
+
+kube_v1_27_3:
+  labels:
+    monitoring_enabled: "false"
+    kube_dashboard_enabled: "true"
+    capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
+    octavia_provider: "{{ magnum_octavia_provider }}"
+  external_network_id: "{{ magnum_external_network }}"
+  master_flavor: "{{ magnum_flavor }}"
+  flavor: "{{ magnum_flavor}}"
+  image: "ubuntu-focal-kube-v1.27.3"
+  name: "kubernetes-v1.27.3"
+  coe: "kubernetes"
+  network_driver: "calico"
+  master_lb_enabled: True
+  floating_ip_enabled: True
+  # Magnum’s default value for dns_nameserver is 8.8.8.8.
+  dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
+  public: True
+
+# List of magnum cluster templates. Format is as required by the
+# stackhpc.os-container-clusters role.
+openstack_container_clusters_templates:
+  - "{{ kube_v1_25_11 }}"
+  - "{{ kube_v1_26_6 }}"
+  - "{{ kube_v1_27_3 }}"

examples/templates/capi-images-templates.j2

@@ -0,0 +1,20 @@
+###############################################################################
+# Magnum container clusters shared variables.
+
+# Flavor must have a minimum of 2 VCPUs
+magnum_flavor: {{ magnum_flavor_name }}
+
+# Network to create tenant cluster FIPs on 
+magnum_external_network: {{ magnum_external_net_name }}
+
+# Provider for cluster loadbalancers
+magnum_octavia_provider: {{ magnum_loadbalancer_provider }}
+
+# helm chart version to use for tenant clusters
+magnum_helm_chart_version: {{ magnum_helm_chart_version }}
+
+{% include './old-images.j2' %}
+{% include './images.j2' %}
+
+{% include './old-templates.j2' %}
+{% include './templates.j2' %}

examples/templates/images.j2

@@ -0,0 +1,28 @@
+###############################################################################
+# Configuration of Glance software images.
+
+{% for item in new_template_data %}
+# Image for {{ item.key }}
+{{ item.value.name }}:
+  name: "{{ item.value.name }}"
+  type: qcow2
+  image_url: "{{ item.value.url }}"
+  is_public: True
+  properties:
+    os_distro: "ubuntu"
+    os_version: "20.04"
+    kube_version: "{{ item.value.kubernetes_version }}"
+
+{% endfor %}
+# List of Kubernetes images. Format is as required by the stackhpc.os-images role.
+kubernetes_images:
+{% if matching_images is defined and matching_temps | length > 0 %}
+{% for item in matching_images %}
+{% if item.value.name not in new_template_data | map(attribute='value.name') %}
+  - "{{ '{{ ' + item.key + ' }}' }}"
+{% endif %}
+{% endfor %}
+{% endif %}
+{% for item in new_template_data %}
+  - "{{ '{{ ' + item.value.name + ' }}' }}"
+{% endfor %}

examples/templates/old-images.j2

@@ -0,0 +1,21 @@
+{% if matching_images is defined and matching_temps | length > 0 %}
+###############################################################################
+# Old magnum images - hide until out of use
+
+{% for item in matching_images %}
+{% if item.value.name not in new_template_data | map(attribute='value.name') %}
+{{ item.key }}:
+{% for key, value in item.value.items() %}
+{% if value is mapping %}
+  {{ key }}:
+{% for k, v in value.items() %}
+    {{ k }}: "{{ v }}"
+{% endfor %}
+{% else %}
+  {{ key }}: "{{ value }}"
+{% endif %}
+{% endfor %}
+
+{% endif %}
+{% endfor %}
+{% endif %}

examples/templates/old-templates.j2

@@ -0,0 +1,26 @@
+{% if matching_temps is defined and matching_temps | length > 0 %}
+###############################################################################
+# Old magnum templates - hide until out of use
+
+{% for item in matching_temps %}
+{% if item.key not in new_template_data | map(attribute='key') %}
+{{ item.key }}:
+{% for key, value in item.value.items() %}
+{% if key == 'is_hidden' %}
+  {{ key }}: {{ value }}
+{% elif value is mapping %}
+  {{ key }}:
+{% for k, v in value.items() %}
+    {{ k }}: "{{ v }}"
+{% endfor %}
+{% else %}
+  {{ key }}: "{{ value }}"
+{% endif %}
+{% endfor %}
+{% if 'is_hidden' not in item.value %}
+is_hidden: True
+{% endif %}
+
+{% endif %}
+{% endfor %}
+{% endif %}

examples/templates/templates.j2

@@ -0,0 +1,36 @@
+###############################################################################
+# Configuration of Magnum container clusters.
+
+{% for item in new_template_data %}
+{{ item.key }}:
+  labels:
+    monitoring_enabled: "true"
+    kube_dashboard_enabled: "true"
+    capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
+    octavia_provider: {{ magnum_loadbalancer_provider }}
+  external_network_id: {{ magnum_external_net_name }}
+  master_flavor: {{ magnum_flavor_name }}
+  flavor: {{ magnum_flavor_name }}
+  image: "{{ item.value.name }}"
+  name: "{{ item.key }}"
+  coe: "kubernetes"
+  network_driver: "calico"
+  master_lb_enabled: True
+  floating_ip_enabled: True
+  dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
+  public: True
+
+{% endfor %}
+# List of magnum cluster templates. Format is as required by the
+# stackhpc.os-container-clusters role.
+openstack_container_clusters_templates:
+{% if matching_temps is defined and matching_temps | length > 0 %}
+{% for item in matching_temps %}
+{% if item.key not in new_template_data | map(attribute='key') %}
+  - "{{ '{{ ' + item.key + ' }}' }}"
+{% endif %}
+{% endfor %}
+{% endif %}
+{% for item in new_template_data %}
+  - "{{ '{{ ' + item.key + ' }}' }}"
+{% endfor %}

requirements.yml

@@ -9,4 +9,4 @@ roles:
 
 collections:
   - name: openstack.cloud
-    version: '<2'
+    version: 2.1.0

tools/merge_config/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+inventory = ./inventory

tools/merge_config/bin/activate

@@ -0,0 +1,22 @@
+#####
+# This script activates the specified environment
+#
+# It needs to be sourced rather than just executed as it sets environment variables
+# for the current shell
+#####
+export CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))"
+
+# If clouds.yaml provided in root dir, then prefer this
+if [ -f "$CONFIG_ROOT/tools/merge_config/clouds.yaml" ]; then
+  export OS_CLOUD="${OS_CLOUD:-"openstack"}"
+  export OS_CLIENT_CONFIG_FILE="$CONFIG_ROOT/tools/merge_config/clouds.yaml"
+fi
+
+# If a Python virtualenv exists, activate it
+VENV="${VENV:-"$CONFIG_ROOT/tools/merge_config/.venv"}"
+if [ -f "$VENV/bin/activate" ]; then
+  echo "Activating Python venv at $VENV"
+  source "$VENV/bin/activate"
+fi
+
+echo "Activated environment"

tools/merge_config/bin/ensure_venv

@@ -0,0 +1,42 @@
+#####
+# This script creates a virtualenv (if not already in one) and installs the required dependencies
+#####
+
+if [[ "$VIRTUAL_ENV" == "" ]]; then
+  # Check python version
+  # NOTE: Python 3.8 or newer is required for ansible 2.12 
+  # which is in turn required for the 'undef' ansible keyword
+  PY_MAJOR=3
+  PY_MINOR=8
+  version_check() {
+    python3 <<EOF
+    import sys
+    if sys.version_info[0] < $PY_MAJOR or sys.version_info[1] < $PY_MINOR:
+        sys.exit(1)
+EOF
+  }
+
+  if ! version_check; then
+    echo "Minimum required python version is $PY_MAJOR.$PY_MINOR" 1>&2
+    echo "Please install a supported version then try again" 1>&2
+    exit 1
+  fi
+
+  VENV="$CONFIG_ROOT/tools/merge_magnum_config/.venv"
+  if [ ! -d "$VENV" ]; then
+    echo "Creating virtual environment at $VENV"
+    python3 -m venv "$VENV"
+fi
+
+else
+VENV=$VIRTUAL_ENV
+fi 
+
+CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))"
+
+echo "Upgrading pip"
+"$VENV/bin/python" -m pip install -U pip
+
+echo "Installing requirements"
+"$VENV/bin/python" -m pip install -r "$CONFIG_ROOT/requirements.txt"
+"$VENV/bin/python" -m pip install -r "$CONFIG_ROOT/tools/merge_config/requirements.txt"

tools/merge_config/bin/run

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))"
+
+source $CONFIG_ROOT/tools/merge_config/bin/activate
+
+export ANSIBLE_CONFIG=$CONFIG_ROOT/tools/merge_config/ansible.cfg
+
+ansible-playbook $CONFIG_ROOT/tools/merge_config/merge_templates.yml

tools/merge_config/clouds.yaml

@@ -0,0 +1,14 @@
+# This clouds.yaml is used to connect to the OpenStack project for the environment
+# It should contain an application credential
+#
+# WARNING: This file should be encrypted
+clouds:
+  openstack:
+    auth:
+      auth_url: https://auth.os-api.cci1.ecmwf.int:443
+      application_credential_id: "2351c754c2684860beb4e8dbdcf63f37"
+      application_credential_secret: "MhssM5w-n2u_YvSI9c2L52u3bZ1kcSE9MZZoiZ-7v1FflhMizSzAxch3HCNKZ3gEOJzRuqsodP6KHszU5znMmw"
+    region_name: "RegionOne"
+    interface: "public"
+    identity_api_version: 3
+    auth_type: "v3applicationcredential"