Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

magnum capi images & templates #13

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
7 changes: 1 addition & 6 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,12 @@ packages. For example:

$ virtualenv venv
$ source venv/bin/activate
$ pip install -U pip
$ python -m pip install --upgrade pip
$ pip install -r requirements.txt

Install Ansible role and collection dependencies from Ansible Galaxy:

.. code-block::

$ ansible-galaxy role install \
-p ansible/roles \
-r requirements.yml

$ ansible-galaxy collection install \
-p ansible/collections \
-r requirements.yml
Expand Down
8 changes: 8 additions & 0 deletions ansible/group_vars/all/openstack
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
###############################################################################
# Configuration of OpenStack user environment for OpenStack.

# List of OpenStack domains. Format is as required by the stackhpc.os-projects
# role.
openstack_domains: []

# List of OpenStack projects. Format is as required by the stackhpc.os-projects
# role.
openstack_projects: []
Expand All @@ -21,6 +25,10 @@ openstack_routers: []
# stackhpc.os-networks role.
openstack_security_groups: []

# List of RBAC definitions in the openstack projct. Format is as required by the
# stackhpc.os-networks role.
openstack_networks_rbac: []

###############################################################################
# Configuration of nova flavors for OpenStack.

Expand Down
2 changes: 2 additions & 0 deletions ansible/group_vars/all/venv
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
---
# Path to create virtualenv in which to install python dependencies.
openstack_venv: "{{ ansible_env.PWD }}/openstack-config-venv"
# Separate venv for roles that require sdk>=1.0 in case it breaks anything
openstack_sdk_1_0_venv: "{{ ansible_env.PWD }}/openstack-1.0-venv"
2 changes: 1 addition & 1 deletion ansible/openstack-container-clusters.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
tags:
- container-clusters-templates
roles:
- role: stackhpc.os-container-clusters
- role: stackhpc.openstack.os_container_clusters
os_container_clusters_venv: "{{ openstack_venv }}"
os_container_clusters_auth_type: "{{ openstack_auth_type }}"
os_container_clusters_auth: "{{ openstack_auth }}"
Expand Down
2 changes: 1 addition & 1 deletion ansible/openstack-flavors.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
tags:
- flavors
roles:
- role: stackhpc.os-flavors
- role: stackhpc.openstack.os_flavors
os_flavors_venv: "{{ openstack_venv }}"
os_flavors_auth_type: "{{ openstack_auth_type }}"
os_flavors_auth: "{{ openstack_auth }}"
Expand Down
2 changes: 1 addition & 1 deletion ansible/openstack-host-aggregates.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
tags:
- host_aggregates
roles:
- role: stackhpc.os_host_aggregates
- role: stackhpc.openstack.os_host_aggregates
os_host_aggregates_venv: "{{ openstack_venv }}"
os_host_aggregates_auth_type: "{{ openstack_auth_type }}"
os_host_aggregates_auth: "{{ openstack_auth }}"
Expand Down
2 changes: 1 addition & 1 deletion ansible/openstack-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
tags:
- images
roles:
- role: stackhpc.os-images
- role: stackhpc.openstack.os_images
os_images_venv: "{{ openstack_venv }}"
os_images_cache: "{{ ansible_env.PWD }}/openstack-config-image-cache"
os_images_auth_type: "{{ openstack_auth_type }}"
Expand Down
3 changes: 2 additions & 1 deletion ansible/openstack-networks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
tags:
- networks
roles:
- role: stackhpc.os-networks
- role: stackhpc.openstack.os_networks
os_networks_venv: "{{ openstack_venv }}"
os_networks_auth_type: "{{ openstack_auth_type }}"
os_networks_auth: "{{ openstack_auth }}"
os_networks_cacert: "{{ openstack_cacert }}"
os_networks: "{{ openstack_networks }}"
os_networks_routers: "{{ openstack_routers }}"
os_networks_security_groups: "{{ openstack_security_groups }}"
os_networks_rbac: "{{ openstack_networks_rbac }}"
3 changes: 2 additions & 1 deletion ansible/openstack-project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@
tags:
- project
roles:
- role: stackhpc.os-projects
- role: stackhpc.openstack.os_projects
os_projects_venv: "{{ openstack_venv }}"
os_projects_auth_type: "{{ openstack_auth_type }}"
os_projects_admin_auth: "{{ openstack_auth }}"
os_projects_cacert: "{{ openstack_cacert }}"
os_projects: "{{ openstack_projects }}"
os_projects_domains: "{{ openstack_domains }}"
Empty file removed ansible/roles/.keep
Empty file.
35 changes: 35 additions & 0 deletions doc/magnum-template-generation.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Automatic Template Generation for CAPI Driver

1. `git cherry-pick XXXXXX` if necessary (will replace this with final commit hash before this PR is merged)
2. change `openstack_images` list to `glance images` in `etc/openstack/openstack-config.yml` if necessary
3. ensure that `openstack_images` looks like:
```
# Images to be uploaded 
openstack_images: "{{ glance_images + kubernetes_images }}"
```
4. source the openstack-config venv you have set up
5. source the rc.sh file that points to the appropriate cloud, or provide a `clouds.yaml` under `./tools/merge_config`
6. ensure `wget`, `python-octaviaclient` and `python-magnumclient` are installed
7. If you have existing `openstack_container_clusters_templates` defined, move them to `etc/openstack/container-clusters.yml`
8. run `./tools/merge_config/bin/run`
9. check output at `etc/openstack-config/container-clusters.yml`
10. If it all checks out, run:
```
tools/openstack-config -p ansible/openstack-container-clusters.yml -- --vault-password-file ~/.vault-secret -e@etc/openstack-config/container-clusters.yml
```
Or include the same `-e@etc/openstack-config/container-clusters.yml` when you run the entire openstack-config suite.

This must be ran before `os-images` is, if any kubernetes images/templates are being hidden/retired, because the cluster template cannot be hidden after the corresponding image is hidden.

If you run the above command, you can run:
```
tools/openstack-config -p ansible/openstack-images.yml -- --vault-password-file ~/.vault-secret  -e@etc/openstack-config/container-clusters.yml
```
afterwards, to upload/hide any of the kubernetes images.

Note: If you run out of space to store images on the control host, you may need to run this in sections - comment out blocks of images, making sure to remove any cached images under `~/openstack-config/ansible/openstack-config-image-cache/` before moving onto the next block.

Note: If the image cache has been cleared (i.e. the old images being set to hidden no longer exist in the cache), then `openstack.cloud.image` will not recognise that it is the same image and will upload a new one, so you should comment these images out from the list.

Note: Container template hiding does not no-op, due to the current state of the magnum API. If the template is already hidden and you rerun this, you will get the error `ClusterTemplate 4ce13776-da1e-42c9-b6f3-1249363d7a4e is referenced by one or multiple clusters (HTTP 400)`
You may also get an error like `Image ubuntu-focal-kube-v1 (HTTP 400)` , which is because the image has been hidden. If your template is already hidden, this is not a problem, but if it isn't hidden, you will need to unhide the images before the template hiding can run.
14 changes: 12 additions & 2 deletions etc/openstack-config/openstack-config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
###############################################################################
# Configuration of OpenStack projects and users user environment.

# List of OpenStack domains. Format is as required by the stackhpc.os-projects
# role.
#openstack_domains:

# List of OpenStack projects. Format is as required by the stackhpc.os-projects
# role.
#openstack_projects:
Expand All @@ -21,6 +25,10 @@
# Format is as required by the stackhpc.os-networks role.
#openstack_security_groups:

# List of RBAC definitions in the openstack projct. Format is as required by the
# stackhpc.os-networks role.
#openstack_networks_rbac:

###############################################################################
# Configuration of nova flavors.

Expand All @@ -39,8 +47,10 @@
# Configuration of Glance software images.

# List of Glance images. Format is as required by the stackhpc.os-images role.
#openstack_images:
#glance_images:

# Images to be uploaded
#openstack_images: "{{ glance_images + kubernetes_images }}"
# List of Diskimage Builder (DIB) elements paths to include in image builds.
#openstack_image_elements:

Expand All @@ -57,4 +67,4 @@

###############################################################################
# Dummy variable to allow Ansible to accept this file.
workaround_ansible_issue_8743: yes
workaround_ansible_issue_8743: yes
117 changes: 117 additions & 0 deletions examples/capi-templates-images.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
###############################################################################
# Configuration of Glance software images.

# Flavor must have a minimum of 2 VCPUs
magnum_flavor: "m1.small"

# Network to create tenant cluster FIPs on
magnum_external_network: "external"

# Provider for cluster loadbalancers
magnum_octavia_provider: "ovn"

# helm chart version to use for tenant clusters
magnum_helm_chart_version: "0.1.1-dev.0.main.221"

ubuntu_focal_kube_v1_25_11:
name: "ubuntu-focal-kube-v1.25.11"
type: qcow2
image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.25.11-230712-0939.qcow2"
is_public: True
properties:
os_distro: "ubuntu"
os_version: "20.04"
kube_version: "v1.25.11"

ubuntu_focal_kube_v1_26_6:
name: "ubuntu-focal-kube-v1.26.6"
type: qcow2
image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.26.6-230712-1010.qcow2"
is_public: True
properties:
os_distro: "ubuntu"
os_version: "20.04"
kube_version: "v1.26.6"

ubuntu_focal_kube_v1_27_3:
name: "ubuntu-focal-kube-v1.27.3"
type: qcow2
image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.27.3-230712-1021.qcow2"
is_public: True
properties:
os_distro: "ubuntu"
os_version: "20.04"
kube_version: "v1.27.3"

# List of Glance images. Format is as required by the stackhpc.os-images role.
openstack_images:
- "{{ ubuntu_focal_kube_v1_25_11 }}"
- "{{ ubuntu_focal_kube_v1_26_6 }}"
- "{{ ubuntu_focal_kube_v1_27_3 }}"

###############################################################################
# Configuration of Magnum container clusters.

kube_v1_25_11:
labels:
monitoring_enabled: "false"
kube_dashboard_enabled: "true"
capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
octavia_provider: "{{ magnum_octavia_provider }}"
external_network_id: "{{ magnum_external_network }}"
master_flavor: "{{ magnum_flavor }}"
flavor: "{{ magnum_flavor }}"
image: "ubuntu-focal-kube-v1.25.11"
name: "kubernetes-v1.25.11"
coe: "kubernetes"
network_driver: "calico"
master_lb_enabled: True
floating_ip_enabled: True
# Magnum’s default value for dns_nameserver is 8.8.8.8.
dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
public: True

kube_v1_26_6:
labels:
monitoring_enabled: "false"
kube_dashboard_enabled: "true"
capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
octavia_provider: "{{ magnum_octavia_provider }}"
external_network_id: "{{ magnum_external_network }}"
master_flavor: "{{ magnum_flavor }}"
flavor: "{{ magnum_flavor }}"
image: "ubuntu-focal-kube-v1.26.6"
name: "kubernetes-v1.26.6"
coe: "kubernetes"
network_driver: "calico"
master_lb_enabled: True
floating_ip_enabled: True
# Magnum’s default value for dns_nameserver is 8.8.8.8.
dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
public: True

kube_v1_27_3:
labels:
monitoring_enabled: "false"
kube_dashboard_enabled: "true"
capi_helm_chart_version: "{{ magnum_helm_chart_version }}"
octavia_provider: "{{ magnum_octavia_provider }}"
external_network_id: "{{ magnum_external_network }}"
master_flavor: "{{ magnum_flavor }}"
flavor: "{{ magnum_flavor}}"
image: "ubuntu-focal-kube-v1.27.3"
name: "kubernetes-v1.27.3"
coe: "kubernetes"
network_driver: "calico"
master_lb_enabled: True
floating_ip_enabled: True
# Magnum’s default value for dns_nameserver is 8.8.8.8.
dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4"
public: True

# List of magnum cluster templates. Format is as required by the
# stackhpc.os-container-clusters role.
openstack_container_clusters_templates:
- "{{ kube_v1_25_11 }}"
- "{{ kube_v1_26_6 }}"
- "{{ kube_v1_27_3 }}"
29 changes: 29 additions & 0 deletions examples/projects-octavia.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
###############################################################################
# Configuration of OpenStack user environment for OpenStack.

# List of OpenStack projects. Format is as required by the stackhpc.os-projects
# role.
openstack_projects:
- "{{ openstack_service_project }}"

# Definition of the openstack service project. Format is as required by the
# stackhpc.os-projects role. Quotas are set to unlimited to avoid Octavia load
# balancer creation failing on quota limits.
openstack_service_project:
name: service
project_domain: default
user_domain: default
quotas: "{{ openstack_octavia_unlimited_quotas }}"

# Dict of quotas to set for service project when Octavia is used.
openstack_octavia_unlimited_quotas:
cores: -1
fixed_ips: -1
floatingip: -1
injected_file_size: -1
injected_files: -1
instances: -1
ram: -1
security_group: -1
security_group_rule: -1
20 changes: 20 additions & 0 deletions examples/templates/capi-images-templates.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
###############################################################################
# Magnum container clusters shared variables.

# Flavor must have a minimum of 2 VCPUs
magnum_flavor: {{ magnum_flavor_name }}

# Network to create tenant cluster FIPs on
magnum_external_network: {{ magnum_external_net_name }}

# Provider for cluster loadbalancers
magnum_octavia_provider: {{ magnum_loadbalancer_provider }}

# helm chart version to use for tenant clusters
magnum_helm_chart_version: {{ magnum_helm_chart_version }}

{% include './old-images.j2' %}
{% include './images.j2' %}

{% include './old-templates.j2' %}
{% include './templates.j2' %}
30 changes: 30 additions & 0 deletions examples/templates/images.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
###############################################################################
# Configuration of Glance software images.

{% for item in new_template_data %}
# Image for {{ item.key }}
{{ item.value.name | replace('-', '_') | replace('.', '_') }}:
name: "{{ item.value.name }}"
type: qcow2
image_url: "{{ item.value.url }}"
visibility: "community"
properties:
os_distro: "ubuntu"
os_version: "20.04"
kube_version: "{{ item.value.kubernetes_version }}"

{% endfor %}
# List of Kubernetes images. Format is as required by the stackhpc.os-images role.
kubernetes_images:
{% if matching_images is defined and matching_temps | length > 0 %}
# Old Images
{% for item in matching_images %}
{% if (item.value.name | replace('_', '-')) not in (new_template_data | map(attribute='value.name')) %}
- "{{ '{{ ' + item.key + ' }}' }}"
{% endif %}
{% endfor %}
{% endif %}
# New Images
{% for item in new_template_data %}
- "{{ '{{ ' + (item.value.name | replace('-', '_')) | replace('.', '_') + ' }}' }}"
{% endfor %}
Loading