diff --git a/README.rst b/README.rst index a459010..ad0d4a5 100644 --- a/README.rst +++ b/README.rst @@ -17,17 +17,12 @@ packages. For example: $ virtualenv venv $ source venv/bin/activate - $ pip install -U pip + $ python -m pip install --upgrade pip $ pip install -r requirements.txt Install Ansible role and collection dependencies from Ansible Galaxy: .. code-block:: - - $ ansible-galaxy role install \ - -p ansible/roles \ - -r requirements.yml - $ ansible-galaxy collection install \ -p ansible/collections \ -r requirements.yml diff --git a/ansible/group_vars/all/openstack b/ansible/group_vars/all/openstack index 690572b..462f3eb 100644 --- a/ansible/group_vars/all/openstack +++ b/ansible/group_vars/all/openstack @@ -2,6 +2,10 @@ ############################################################################### # Configuration of OpenStack user environment for OpenStack. +# List of OpenStack domains. Format is as required by the stackhpc.os-projects +# role. +openstack_domains: [] + # List of OpenStack projects. Format is as required by the stackhpc.os-projects # role. openstack_projects: [] @@ -21,6 +25,10 @@ openstack_routers: [] # stackhpc.os-networks role. openstack_security_groups: [] +# List of RBAC definitions in the openstack projct. Format is as required by the +# stackhpc.os-networks role. +openstack_networks_rbac: [] + ############################################################################### # Configuration of nova flavors for OpenStack. diff --git a/ansible/group_vars/all/venv b/ansible/group_vars/all/venv index c729e5e..cee2bbd 100644 --- a/ansible/group_vars/all/venv +++ b/ansible/group_vars/all/venv @@ -1,3 +1,5 @@ --- # Path to create virtualenv in which to install python dependencies. openstack_venv: "{{ ansible_env.PWD }}/openstack-config-venv" +# Separate venv for roles that require sdk>=1.0 in case it breaks anything +openstack_sdk_1_0_venv: "{{ ansible_env.PWD }}/openstack-1.0-venv" \ No newline at end of file diff --git a/ansible/openstack-container-clusters.yml b/ansible/openstack-container-clusters.yml index 0194742..5df9a4c 100644 --- a/ansible/openstack-container-clusters.yml +++ b/ansible/openstack-container-clusters.yml @@ -4,7 +4,7 @@ tags: - container-clusters-templates roles: - - role: stackhpc.os-container-clusters + - role: stackhpc.openstack.os_container_clusters os_container_clusters_venv: "{{ openstack_venv }}" os_container_clusters_auth_type: "{{ openstack_auth_type }}" os_container_clusters_auth: "{{ openstack_auth }}" diff --git a/ansible/openstack-flavors.yml b/ansible/openstack-flavors.yml index 16a06b3..69b94f2 100644 --- a/ansible/openstack-flavors.yml +++ b/ansible/openstack-flavors.yml @@ -4,7 +4,7 @@ tags: - flavors roles: - - role: stackhpc.os-flavors + - role: stackhpc.openstack.os_flavors os_flavors_venv: "{{ openstack_venv }}" os_flavors_auth_type: "{{ openstack_auth_type }}" os_flavors_auth: "{{ openstack_auth }}" diff --git a/ansible/openstack-host-aggregates.yml b/ansible/openstack-host-aggregates.yml index 3826904..f37449e 100644 --- a/ansible/openstack-host-aggregates.yml +++ b/ansible/openstack-host-aggregates.yml @@ -4,7 +4,7 @@ tags: - host_aggregates roles: - - role: stackhpc.os_host_aggregates + - role: stackhpc.openstack.os_host_aggregates os_host_aggregates_venv: "{{ openstack_venv }}" os_host_aggregates_auth_type: "{{ openstack_auth_type }}" os_host_aggregates_auth: "{{ openstack_auth }}" diff --git a/ansible/openstack-images.yml b/ansible/openstack-images.yml index 3c89df6..8fafc3c 100644 --- a/ansible/openstack-images.yml +++ b/ansible/openstack-images.yml @@ -4,7 +4,7 @@ tags: - images roles: - - role: stackhpc.os-images + - role: stackhpc.openstack.os_images os_images_venv: "{{ openstack_venv }}" os_images_cache: "{{ ansible_env.PWD }}/openstack-config-image-cache" os_images_auth_type: "{{ openstack_auth_type }}" diff --git a/ansible/openstack-networks.yml b/ansible/openstack-networks.yml index 525712d..5f175d0 100644 --- a/ansible/openstack-networks.yml +++ b/ansible/openstack-networks.yml @@ -4,7 +4,7 @@ tags: - networks roles: - - role: stackhpc.os-networks + - role: stackhpc.openstack.os_networks os_networks_venv: "{{ openstack_venv }}" os_networks_auth_type: "{{ openstack_auth_type }}" os_networks_auth: "{{ openstack_auth }}" @@ -12,3 +12,4 @@ os_networks: "{{ openstack_networks }}" os_networks_routers: "{{ openstack_routers }}" os_networks_security_groups: "{{ openstack_security_groups }}" + os_networks_rbac: "{{ openstack_networks_rbac }}" diff --git a/ansible/openstack-project.yml b/ansible/openstack-project.yml index f18334b..2de1fb9 100644 --- a/ansible/openstack-project.yml +++ b/ansible/openstack-project.yml @@ -4,9 +4,10 @@ tags: - project roles: - - role: stackhpc.os-projects + - role: stackhpc.openstack.os_projects os_projects_venv: "{{ openstack_venv }}" os_projects_auth_type: "{{ openstack_auth_type }}" os_projects_admin_auth: "{{ openstack_auth }}" os_projects_cacert: "{{ openstack_cacert }}" os_projects: "{{ openstack_projects }}" + os_projects_domains: "{{ openstack_domains }}" diff --git a/ansible/roles/.keep b/ansible/roles/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/doc/magnum-template-generation.rst b/doc/magnum-template-generation.rst new file mode 100644 index 0000000..d483dc5 --- /dev/null +++ b/doc/magnum-template-generation.rst @@ -0,0 +1,35 @@ +# Automatic Template Generation for CAPI Driver + +1. `git cherry-pick XXXXXX` if necessary (will replace this with final commit hash before this PR is merged) +2. change `openstack_images` list to `glance images` in `etc/openstack/openstack-config.yml` if necessary +3. ensure that `openstack_images` looks like: +``` +# Images to be uploaded  +openstack_images: "{{ glance_images + kubernetes_images }}" +``` +4. source the openstack-config venv you have set up +5. source the rc.sh file that points to the appropriate cloud, or provide a `clouds.yaml` under `./tools/merge_config` +6. ensure `wget`, `python-octaviaclient` and `python-magnumclient` are installed +7. If you have existing `openstack_container_clusters_templates` defined, move them to `etc/openstack/container-clusters.yml` +8. run `./tools/merge_config/bin/run` +9. check output at `etc/openstack-config/container-clusters.yml` +10. If it all checks out, run: +``` +tools/openstack-config -p ansible/openstack-container-clusters.yml -- --vault-password-file ~/.vault-secret -e@etc/openstack-config/container-clusters.yml +``` +Or include the same `-e@etc/openstack-config/container-clusters.yml` when you run the entire openstack-config suite. + +This must be ran before `os-images` is, if any kubernetes images/templates are being hidden/retired, because the cluster template cannot be hidden after the corresponding image is hidden. + +If you run the above command, you can run: +``` +tools/openstack-config -p ansible/openstack-images.yml -- --vault-password-file ~/.vault-secret  -e@etc/openstack-config/container-clusters.yml +``` +afterwards, to upload/hide any of the kubernetes images. + +Note: If you run out of space to store images on the control host, you may need to run this in sections - comment out blocks of images, making sure to remove any cached images under `~/openstack-config/ansible/openstack-config-image-cache/` before moving onto the next block. + +Note: If the image cache has been cleared (i.e. the old images being set to hidden no longer exist in the cache), then `openstack.cloud.image` will not recognise that it is the same image and will upload a new one, so you should comment these images out from the list. + +Note: Container template hiding does not no-op, due to the current state of the magnum API. If the template is already hidden and you rerun this, you will get the error `ClusterTemplate 4ce13776-da1e-42c9-b6f3-1249363d7a4e is referenced by one or multiple clusters (HTTP 400)` +You may also get an error like `Image ubuntu-focal-kube-v1 (HTTP 400)` , which is because the image has been hidden. If your template is already hidden, this is not a problem, but if it isn't hidden, you will need to unhide the images before the template hiding can run. \ No newline at end of file diff --git a/etc/openstack-config/openstack-config.yml b/etc/openstack-config/openstack-config.yml index fd88313..b4b2731 100644 --- a/etc/openstack-config/openstack-config.yml +++ b/etc/openstack-config/openstack-config.yml @@ -2,6 +2,10 @@ ############################################################################### # Configuration of OpenStack projects and users user environment. +# List of OpenStack domains. Format is as required by the stackhpc.os-projects +# role. +#openstack_domains: + # List of OpenStack projects. Format is as required by the stackhpc.os-projects # role. #openstack_projects: @@ -21,6 +25,10 @@ # Format is as required by the stackhpc.os-networks role. #openstack_security_groups: +# List of RBAC definitions in the openstack projct. Format is as required by the +# stackhpc.os-networks role. +#openstack_networks_rbac: + ############################################################################### # Configuration of nova flavors. @@ -39,8 +47,10 @@ # Configuration of Glance software images. # List of Glance images. Format is as required by the stackhpc.os-images role. -#openstack_images: +#glance_images: +# Images to be uploaded +#openstack_images: "{{ glance_images + kubernetes_images }}" # List of Diskimage Builder (DIB) elements paths to include in image builds. #openstack_image_elements: @@ -57,4 +67,4 @@ ############################################################################### # Dummy variable to allow Ansible to accept this file. -workaround_ansible_issue_8743: yes +workaround_ansible_issue_8743: yes \ No newline at end of file diff --git a/examples/capi-templates-images.yml b/examples/capi-templates-images.yml new file mode 100644 index 0000000..3410b4e --- /dev/null +++ b/examples/capi-templates-images.yml @@ -0,0 +1,117 @@ +############################################################################### +# Configuration of Glance software images. + +# Flavor must have a minimum of 2 VCPUs +magnum_flavor: "m1.small" + +# Network to create tenant cluster FIPs on +magnum_external_network: "external" + +# Provider for cluster loadbalancers +magnum_octavia_provider: "ovn" + +# helm chart version to use for tenant clusters +magnum_helm_chart_version: "0.1.1-dev.0.main.221" + +ubuntu_focal_kube_v1_25_11: + name: "ubuntu-focal-kube-v1.25.11" + type: qcow2 + image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.25.11-230712-0939.qcow2" + is_public: True + properties: + os_distro: "ubuntu" + os_version: "20.04" + kube_version: "v1.25.11" + +ubuntu_focal_kube_v1_26_6: + name: "ubuntu-focal-kube-v1.26.6" + type: qcow2 + image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.26.6-230712-1010.qcow2" + is_public: True + properties: + os_distro: "ubuntu" + os_version: "20.04" + kube_version: "v1.26.6" + +ubuntu_focal_kube_v1_27_3: + name: "ubuntu-focal-kube-v1.27.3" + type: qcow2 + image_url: "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images/ubuntu-focal-kube-v1.27.3-230712-1021.qcow2" + is_public: True + properties: + os_distro: "ubuntu" + os_version: "20.04" + kube_version: "v1.27.3" + +# List of Glance images. Format is as required by the stackhpc.os-images role. +openstack_images: + - "{{ ubuntu_focal_kube_v1_25_11 }}" + - "{{ ubuntu_focal_kube_v1_26_6 }}" + - "{{ ubuntu_focal_kube_v1_27_3 }}" + +############################################################################### +# Configuration of Magnum container clusters. + +kube_v1_25_11: + labels: + monitoring_enabled: "false" + kube_dashboard_enabled: "true" + capi_helm_chart_version: "{{ magnum_helm_chart_version }}" + octavia_provider: "{{ magnum_octavia_provider }}" + external_network_id: "{{ magnum_external_network }}" + master_flavor: "{{ magnum_flavor }}" + flavor: "{{ magnum_flavor }}" + image: "ubuntu-focal-kube-v1.25.11" + name: "kubernetes-v1.25.11" + coe: "kubernetes" + network_driver: "calico" + master_lb_enabled: True + floating_ip_enabled: True + # Magnum’s default value for dns_nameserver is 8.8.8.8. + dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4" + public: True + +kube_v1_26_6: + labels: + monitoring_enabled: "false" + kube_dashboard_enabled: "true" + capi_helm_chart_version: "{{ magnum_helm_chart_version }}" + octavia_provider: "{{ magnum_octavia_provider }}" + external_network_id: "{{ magnum_external_network }}" + master_flavor: "{{ magnum_flavor }}" + flavor: "{{ magnum_flavor }}" + image: "ubuntu-focal-kube-v1.26.6" + name: "kubernetes-v1.26.6" + coe: "kubernetes" + network_driver: "calico" + master_lb_enabled: True + floating_ip_enabled: True + # Magnum’s default value for dns_nameserver is 8.8.8.8. + dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4" + public: True + +kube_v1_27_3: + labels: + monitoring_enabled: "false" + kube_dashboard_enabled: "true" + capi_helm_chart_version: "{{ magnum_helm_chart_version }}" + octavia_provider: "{{ magnum_octavia_provider }}" + external_network_id: "{{ magnum_external_network }}" + master_flavor: "{{ magnum_flavor }}" + flavor: "{{ magnum_flavor}}" + image: "ubuntu-focal-kube-v1.27.3" + name: "kubernetes-v1.27.3" + coe: "kubernetes" + network_driver: "calico" + master_lb_enabled: True + floating_ip_enabled: True + # Magnum’s default value for dns_nameserver is 8.8.8.8. + dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4" + public: True + +# List of magnum cluster templates. Format is as required by the +# stackhpc.os-container-clusters role. +openstack_container_clusters_templates: + - "{{ kube_v1_25_11 }}" + - "{{ kube_v1_26_6 }}" + - "{{ kube_v1_27_3 }}" diff --git a/examples/projects-octavia.yml b/examples/projects-octavia.yml new file mode 100644 index 0000000..f116b18 --- /dev/null +++ b/examples/projects-octavia.yml @@ -0,0 +1,29 @@ +--- +############################################################################### +# Configuration of OpenStack user environment for OpenStack. + +# List of OpenStack projects. Format is as required by the stackhpc.os-projects +# role. +openstack_projects: + - "{{ openstack_service_project }}" + +# Definition of the openstack service project. Format is as required by the +# stackhpc.os-projects role. Quotas are set to unlimited to avoid Octavia load +# balancer creation failing on quota limits. +openstack_service_project: + name: service + project_domain: default + user_domain: default + quotas: "{{ openstack_octavia_unlimited_quotas }}" + +# Dict of quotas to set for service project when Octavia is used. +openstack_octavia_unlimited_quotas: + cores: -1 + fixed_ips: -1 + floatingip: -1 + injected_file_size: -1 + injected_files: -1 + instances: -1 + ram: -1 + security_group: -1 + security_group_rule: -1 diff --git a/examples/templates/capi-images-templates.j2 b/examples/templates/capi-images-templates.j2 new file mode 100644 index 0000000..5bbd493 --- /dev/null +++ b/examples/templates/capi-images-templates.j2 @@ -0,0 +1,20 @@ +############################################################################### +# Magnum container clusters shared variables. + +# Flavor must have a minimum of 2 VCPUs +magnum_flavor: {{ magnum_flavor_name }} + +# Network to create tenant cluster FIPs on +magnum_external_network: {{ magnum_external_net_name }} + +# Provider for cluster loadbalancers +magnum_octavia_provider: {{ magnum_loadbalancer_provider }} + +# helm chart version to use for tenant clusters +magnum_helm_chart_version: {{ magnum_helm_chart_version }} + +{% include './old-images.j2' %} +{% include './images.j2' %} + +{% include './old-templates.j2' %} +{% include './templates.j2' %} diff --git a/examples/templates/images.j2 b/examples/templates/images.j2 new file mode 100644 index 0000000..2d58b19 --- /dev/null +++ b/examples/templates/images.j2 @@ -0,0 +1,30 @@ +############################################################################### +# Configuration of Glance software images. + +{% for item in new_template_data %} +# Image for {{ item.key }} +{{ item.value.name | replace('-', '_') | replace('.', '_') }}: + name: "{{ item.value.name }}" + type: qcow2 + image_url: "{{ item.value.url }}" + visibility: "community" + properties: + os_distro: "ubuntu" + os_version: "20.04" + kube_version: "{{ item.value.kubernetes_version }}" + +{% endfor %} +# List of Kubernetes images. Format is as required by the stackhpc.os-images role. +kubernetes_images: +{% if matching_images is defined and matching_temps | length > 0 %} +# Old Images +{% for item in matching_images %} +{% if (item.value.name | replace('_', '-')) not in (new_template_data | map(attribute='value.name')) %} + - "{{ '{{ ' + item.key + ' }}' }}" +{% endif %} +{% endfor %} +{% endif %} +# New Images +{% for item in new_template_data %} + - "{{ '{{ ' + (item.value.name | replace('-', '_')) | replace('.', '_') + ' }}' }}" +{% endfor %} diff --git a/examples/templates/old-images.j2 b/examples/templates/old-images.j2 new file mode 100644 index 0000000..992db29 --- /dev/null +++ b/examples/templates/old-images.j2 @@ -0,0 +1,24 @@ +{% if matching_images is defined and matching_temps | length > 0 %} +############################################################################### +# Old magnum images - hide until out of use + +{% for item in matching_images %} +{% if (item.value.name | replace('_', '-')) not in new_template_data | map(attribute='value.name') %} +{{ item.key }}: +{% for key, value in item.value.items() %} +{% if value is mapping %} + {{ key }}: +{% for k, v in value.items() %} + {{ k }}: "{{ v }}" +{% endfor %} +{% else %} + {{ key }}: "{{ value }}" +{% endif %} +{% endfor %} +{% if 'hide_image' not in item.value %} + hide_image: "True" +{% endif %} + +{% endif %} +{% endfor %} +{% endif %} diff --git a/examples/templates/old-templates.j2 b/examples/templates/old-templates.j2 new file mode 100644 index 0000000..56c3767 --- /dev/null +++ b/examples/templates/old-templates.j2 @@ -0,0 +1,26 @@ +{% if matching_temps is defined and matching_temps | length > 0 %} +############################################################################### +# Old magnum templates - hide until out of use + +{% set versions_to_check = new_template_data | map(attribute='value.kubernetes_version') | map('replace', '.', '-') %} +{% set keys_and_versions = new_template_data | map(attribute='key') | zip(versions_to_check) %} +{% for item in matching_temps %} +{% if (item.key | replace('_', '-')) not in keys_and_versions | map('join', '-') %} +{{ item.key }}: +{% for key, value in item.value.items() %} +{% if value is mapping %} + {{ key }}: +{% for k, v in value.items() %} + {{ k }}: "{{ v }}" +{% endfor %} +{% else %} + {{ key }}: "{{ value }}" +{% endif %} +{% endfor %} +{% if 'is_hidden' not in item.value %} + is_hidden: True +{% endif %} + +{% endif %} +{% endfor %} +{% endif %} diff --git a/examples/templates/templates.j2 b/examples/templates/templates.j2 new file mode 100644 index 0000000..fa627e7 --- /dev/null +++ b/examples/templates/templates.j2 @@ -0,0 +1,40 @@ +############################################################################### +# Configuration of Magnum container clusters. + +{% for item in new_template_data %} +{{ item.key | replace('-', '_') }}_{{ item.value.kubernetes_version | replace('.', '_') }}: + labels: + monitoring_enabled: "True" + kube_dashboard_enabled: "True" + capi_helm_chart_version: "{{ magnum_helm_chart_version }}" + octavia_provider: {{ magnum_loadbalancer_provider }} + external_network_id: {{ magnum_external_net_name }} + master_flavor: {{ magnum_flavor_name }} + flavor: {{ magnum_flavor_name }} + image: "{{ item.value.name }}" + name: "{{ item.key }}" + coe: "kubernetes" + network_driver: "calico" + master_lb_enabled: "True" + floating_ip_enabled: "True" + dns_nameserver: "1.1.1.1,8.8.8.8,8.8.4.4" + public: True + +{% endfor %} +# List of magnum cluster templates. Format is as required by the +# stackhpc.os-container-clusters role. +openstack_container_clusters_templates: +{% if matching_temps is defined and matching_temps | length > 0 %} +# Old Templates +{% set versions_to_check = new_template_data | map(attribute='value.kubernetes_version') | map('replace', '.', '-') %} +{% set keys_and_versions = new_template_data | map(attribute='key') | zip(versions_to_check) %} +{% for item in matching_temps %} +{% if (item.key | replace('_', '-')) not in keys_and_versions | map('join', '-') %} + - "{{ '{{ ' + item.key + ' }}' }}" +{% endif %} +{% endfor %} +{% endif %} +# New Templates +{% for item in new_template_data %} + - "{{ '{{ ' + (item.key| replace('-', '_')) + '_' + (item.value.kubernetes_version | replace('.', '_')) + ' }}' }}" +{% endfor %} diff --git a/requirements.txt b/requirements.txt index bcefbee..0c0c0b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -# Use Ansible 5 for consistent Rocky 9 behaviour when available, otherwise use Ansible 4 +# Use Ansible 5 for consistent Rocky 9 behaviour when available, otherwise use +# Ansible 4 ansible>=4,<5; python_version<"3.7" -ansible>=5,<6; python_version>="3.7" \ No newline at end of file +ansible>=5,<6; python_version>="3.7" diff --git a/requirements.yml b/requirements.yml index fda082e..99026c8 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,12 +1,6 @@ --- -roles: - - name: stackhpc.os-flavors - - name: stackhpc.os-images - - name: stackhpc.os-networks - - name: stackhpc.os-projects - - name: stackhpc.os_host_aggregates - - name: stackhpc.os-container-clusters - collections: - name: openstack.cloud - version: '<2' + version: 2.1.0 + - name: stackhpc.openstack + version: 0.0.1 diff --git a/tools/merge_config/ansible.cfg b/tools/merge_config/ansible.cfg new file mode 100644 index 0000000..154ac15 --- /dev/null +++ b/tools/merge_config/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +inventory = ./inventory \ No newline at end of file diff --git a/tools/merge_config/bin/activate b/tools/merge_config/bin/activate new file mode 100755 index 0000000..7c8854b --- /dev/null +++ b/tools/merge_config/bin/activate @@ -0,0 +1,27 @@ +##### +# This script activates the specified environment +# +# It needs to be sourced rather than just executed as it sets environment variables +# for the current shell +##### +export CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))" + +# If clouds.yaml provided in root dir, then prefer this +if [ -f "$CONFIG_ROOT/tools/merge_config/clouds.yaml" ]; then + export OS_CLOUD="${OS_CLOUD:-"openstack"}" + export OS_CLIENT_CONFIG_FILE="$CONFIG_ROOT/tools/merge_config/clouds.yaml" +fi + +# If a Python virtualenv exists, activate it +if [[ "$VIRTUAL_ENV" != "" ]]; then + VENV=$VIRTUAL_ENV +fi + +VENV="${VENV:-"$CONFIG_ROOT/tools/merge_config/.venv"}" + +if [ -f "$VENV/bin/activate" ]; then + echo "Activating Python venv at $VENV" + source "$VENV/bin/activate" +fi + +echo "Activated environment" diff --git a/tools/merge_config/bin/ensure_venv b/tools/merge_config/bin/ensure_venv new file mode 100755 index 0000000..e6b71c5 --- /dev/null +++ b/tools/merge_config/bin/ensure_venv @@ -0,0 +1,42 @@ +##### +# This script creates a virtualenv (if not already in one) and installs the required dependencies +##### + +CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))" + +if [[ "$VIRTUAL_ENV" == "" ]]; then + # Check python version + # NOTE: Python 3.8 or newer is required for ansible 2.12 + # which is in turn required for the 'undef' ansible keyword + PY_MAJOR=3 + PY_MINOR=8 + version_check() { + python3 <&2 + echo "Please install a supported version then try again" 1>&2 + exit 1 + fi + + VENV="$CONFIG_ROOT/tools/merge_config/.venv" + if [ ! -d "$VENV" ]; then + echo "Creating virtual environment at $VENV" + python3 -m venv "$VENV" +fi + +else +VENV=$VIRTUAL_ENV +fi + +echo "Upgrading pip" +"$VENV/bin/python" -m pip install -U pip + +echo "Installing requirements" +"$VENV/bin/python" -m pip install -r "$CONFIG_ROOT/requirements.txt" +"$VENV/bin/python" -m pip install -r "$CONFIG_ROOT/tools/merge_config/requirements.txt" \ No newline at end of file diff --git a/tools/merge_config/bin/run b/tools/merge_config/bin/run new file mode 100755 index 0000000..d725d7f --- /dev/null +++ b/tools/merge_config/bin/run @@ -0,0 +1,9 @@ +#!/bin/bash + +CONFIG_ROOT="$(dirname $(dirname $(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))))" + +source $CONFIG_ROOT/tools/merge_config/bin/activate + +export ANSIBLE_CONFIG=$CONFIG_ROOT/tools/merge_config/ansible.cfg + +ansible-playbook $CONFIG_ROOT/tools/merge_config/merge_templates.yml diff --git a/tools/merge_config/inventory/group_vars/all/variables.yml b/tools/merge_config/inventory/group_vars/all/variables.yml new file mode 100644 index 0000000..ad97388 --- /dev/null +++ b/tools/merge_config/inventory/group_vars/all/variables.yml @@ -0,0 +1,41 @@ + +_magnum_flavor_name: >- + {{ + lookup('pipe', 'openstack flavor list -f json') | + from_json | + selectattr('Disk', '>=', 20) | + selectattr('VCPUs', '>=', 2) | + selectattr('VCPUs', '<=', 8) | + selectattr('RAM','>=',2048) | + selectattr('RAM','<=',8192) | + first | + json_query('Name') + }} + +magnum_flavor_name: "{{ _magnum_flavor_name }}" + +_magnum_external_net_name: >- + {{ + lookup('pipe', 'openstack network list --external -f json') | + from_json | + map(attribute='Name') | + reject('search', 'ceph') | + first + }} + +magnum_external_net_name: "{{ _magnum_external_net_name }}" + +_magnum_loadbalancer_provider: >- + {{- + lookup('pipe', 'openstack loadbalancer provider list -f json') | + from_json | + map(attribute = 'name') | + reject('equalto', 'octavia') | + select('match','ovn') | + default('amphora') | + first + }} + +magnum_loadbalancer_provider: "{{ _magnum_loadbalancer_provider }}" + +magnum_helm_chart_version: "{{ helm_chart_version.stdout }}" \ No newline at end of file diff --git a/tools/merge_config/inventory/hosts b/tools/merge_config/inventory/hosts new file mode 100644 index 0000000..4cf0cae --- /dev/null +++ b/tools/merge_config/inventory/hosts @@ -0,0 +1 @@ +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/tools/merge_config/merge_templates.yml b/tools/merge_config/merge_templates.yml new file mode 100644 index 0000000..0a8579b --- /dev/null +++ b/tools/merge_config/merge_templates.yml @@ -0,0 +1,63 @@ +--- +- hosts: localhost + vars: + root_dir: "{{ lookup('env','CONFIG_ROOT') }}" + site_vars_file: "{{ root_dir }}/etc/openstack-config/container-clusters.yml" + + tasks: + - name: Check if cluster containers file exists + stat: + path: "{{ site_vars_file }}" + register: file_stat + + - name: Load site_vars from file if it exists, otherwise set to an empty dictionary + set_fact: + site_vars: "{{ lookup('file', site_vars_file) | from_yaml }}" + when: file_stat.stat.exists + + - name: Set site_vars to an empty dictionary if the file doesn't exist + set_fact: + site_vars: {} + when: not file_stat.stat.exists + + - name: Find old templates + set_fact: + matching_temps: "{{ site_vars | dict2items | selectattr('key', 'match', 'kube*') | rejectattr('key','match','kubernetes_images') | list }}" + when: site_vars is defined and site_vars | length > 0 + + - name: Find old images + set_fact: + matching_images: "{{ site_vars | dict2items | selectattr('key', 'match', 'ubuntu*') | list }}" + when: site_vars is defined and site_vars | length > 0 + + - name: Fetch capi-helm-charts version + shell: "curl -s --url \"https://api.github.com/repos/stackhpc/capi-helm-charts/releases/latest\" | grep tag_name | awk -F '\"' '{print $4}' " + register: helm_chart_version + changed_when: false + + - name: Fetch dependencies.json using wget + shell: 'wget -O - "https://github.com/stackhpc/capi-helm-charts/releases/download/{{ helm_chart_version.stdout }}/dependencies.json" ' + register: dependencies_response + changed_when: false + + - name: Parse JSON response + set_fact: + dependencies: "{{ dependencies_response.stdout | from_json | dict2items | list }}" + + - name: Fetch manifest.json using wget + shell: "wget -O - https://github.com/stackhpc/azimuth-images/releases/download/{{ dependencies | json_query('[?key==`azimuth-images`].value | [0]') }}/manifest.json" + register: manifest_response + changed_when: false + + - name: Parse JSON response + set_fact: + new_template_data: "{{ manifest_response.stdout | from_json | dict2items | selectattr('key', 'match', 'kubernetes*') | list }}" + + - debug: + var: new_template_data + + - name: Template images & templates + template: + src: "{{ root_dir }}/examples/templates/capi-images-templates.j2" + dest: "{{ root_dir }}/etc/openstack-config/container-clusters.yml" + diff --git a/tools/merge_config/requirements.txt b/tools/merge_config/requirements.txt new file mode 100644 index 0000000..54a82f5 --- /dev/null +++ b/tools/merge_config/requirements.txt @@ -0,0 +1,5 @@ +jmespath +python-openstackclient +python-magnumclient +python-octaviaclient +munch \ No newline at end of file