Add support for Ceph RadosGW integration

* Register Swift-compatible endpoints in Keystone
* Load balance across RadosGW API servers using HAProxy

The support is exercised in the cephadm CI jobs, but since RGW is
not currently enabled via cephadm, it is not yet tested.

https://docs.ceph.com/en/latest/radosgw/keystone/

Implements: blueprint ceph-rgw

Change-Id: I891c3ed4ed93512607afe65a42dd99596fd4dbf9
(cherry picked from commit 5b75fefb7d9f50c197f3cd86e18059cb31ca9984)
(cherry picked from commit 31f18de)

ansible/group_vars/all.yml

@@ -270,6 +270,10 @@ blazar_api_port: "1234"
 
 caso_tcp_output_port: "24224"
 
+ceph_rgw_internal_fqdn: "{{ kolla_internal_fqdn }}"
+ceph_rgw_external_fqdn: "{{ kolla_external_fqdn }}"
+ceph_rgw_port: "6780"
+
 cinder_internal_fqdn: "{{ kolla_internal_fqdn }}"
 cinder_external_fqdn: "{{ kolla_external_fqdn }}"
 cinder_api_port: "8776"
@@ -589,6 +593,8 @@ enable_ceilometer: "no"
 enable_ceilometer_ipmi: "no"
 enable_cells: "no"
 enable_central_logging: "no"
+enable_ceph_rgw: "no"
+enable_ceph_rgw_loadbalancer: "{{ enable_ceph_rgw | bool }}"
 enable_chrony: "yes"
 enable_cinder: "no"
 enable_cinder_backup: "yes"

ansible/roles/ceph-rgw/defaults/main.yml

@@ -0,0 +1,78 @@
+---
+project_name: "ceph-rgw"
+
+ceph_rgw_services:
+  # NOTE(mgoddard): There is no container deployment, this is used for load
+  # balancer configuration.
+  ceph-rgw:
+    group: "all"
+    enabled: "{{ enable_ceph_rgw | bool }}"
+    haproxy:
+      radosgw:
+        enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}"
+        mode: "http"
+        external: false
+        port: "{{ ceph_rgw_port }}"
+        custom_member_list: "{{ ceph_rgw_haproxy_members }}"
+      radosgw_external:
+        enabled: "{{ enable_ceph_rgw_loadbalancer | bool }}"
+        mode: "http"
+        external: true
+        port: "{{ ceph_rgw_port }}"
+        custom_member_list: "{{ ceph_rgw_haproxy_members }}"
+
+####################
+# Load balancer
+####################
+
+# List of Ceph RadosGW hostname:port to use as HAProxy backends.
+ceph_rgw_hosts: []
+ceph_rgw_haproxy_members: "{{ ceph_rgw_hosts | map('regex_replace', '(.*)', 'server \\1 \\1 ' + ceph_rgw_haproxy_healthcheck) | list }}"
+ceph_rgw_haproxy_healthcheck: "check inter 2000 rise 2 fall 5"
+
+
+####################
+# OpenStack
+####################
+
+# Whether to register Ceph RadosGW swift-compatible endpoints in Keystone.
+enable_ceph_rgw_keystone: "{{ enable_ceph_rgw | bool }}"
+
+# Enable/disable ceph-rgw compatibility with OpenStack Swift.
+ceph_rgw_compatibility: false
+
+# Enable/disable including the account (project) in the endpoint URL. This
+# allows for cross-project and public object access.
+ceph_rgw_account_in_url: false
+
+ceph_rgw_endpoint_path: "{{ '/' if ceph_rgw_compatibility | bool else '/swift/' }}v1{% if ceph_rgw_account_in_url | bool %}/AUTH_%(project_id)s{% endif %}"
+
+ceph_rgw_admin_endpoint: "{{ admin_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+ceph_rgw_internal_endpoint: "{{ internal_protocol }}://{{ ceph_rgw_internal_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+ceph_rgw_public_endpoint: "{{ public_protocol }}://{{ ceph_rgw_external_fqdn | put_address_in_context('url') }}:{{ ceph_rgw_port }}{{ ceph_rgw_endpoint_path }}"
+
+ceph_rgw_keystone_user: "ceph_rgw"
+
+openstack_ceph_rgw_auth: "{{ openstack_auth }}"
+
+
+####################
+# Keystone
+####################
+ceph_rgw_ks_services:
+  - name: "swift"
+    type: "object-store"
+    description: "Openstack Object Storage"
+    endpoints:
+      - {'interface': 'admin', 'url': '{{ ceph_rgw_admin_endpoint }}'}
+      - {'interface': 'internal', 'url': '{{ ceph_rgw_internal_endpoint }}'}
+      - {'interface': 'public', 'url': '{{ ceph_rgw_public_endpoint }}'}
+
+ceph_rgw_ks_users:
+  - project: "service"
+    user: "{{ ceph_rgw_keystone_user }}"
+    password: "{{ ceph_rgw_keystone_password }}"
+    role: "admin"
+
+ceph_rgw_ks_roles:
+  - "ResellerAdmin"

ansible/roles/ceph-rgw/tasks/check.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/config.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/deploy-containers.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/deploy.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: register.yml

ansible/roles/ceph-rgw/tasks/loadbalancer.yml

@@ -0,0 +1,7 @@
+---
+- name: "Configure haproxy for {{ project_name }}"
+  import_role:
+    role: haproxy-config
+  vars:
+    project_services: "{{ ceph_rgw_services }}"
+  tags: always

ansible/roles/ceph-rgw/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ kolla_action }}.yml"

ansible/roles/ceph-rgw/tasks/precheck.yml

@@ -0,0 +1,10 @@
+---
+- name: Fail if load balancer members not set
+  fail:
+    msg: >-
+      Ceph RadosGW load balancer configuration is enabled
+      (enable_ceph_rgw_loadbalancer) but no HAProxy members are configured.
+      Have you set ceph_rgw_hosts?
+  when:
+    - enable_ceph_rgw_loadbalancer | bool
+    - ceph_rgw_haproxy_members | length == 0

ansible/roles/ceph-rgw/tasks/pull.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: deploy.yml

ansible/roles/ceph-rgw/tasks/register.yml

@@ -0,0 +1,9 @@
+---
+- import_role:
+    name: service-ks-register
+  vars:
+    service_ks_register_auth: "{{ openstack_ceph_rgw_auth }}"
+    service_ks_register_services: "{{ ceph_rgw_ks_services }}"
+    service_ks_register_users: "{{ ceph_rgw_ks_users }}"
+    service_ks_register_roles: "{{ ceph_rgw_ks_roles }}"
+  when: enable_ceph_rgw_keystone | bool

ansible/roles/ceph-rgw/tasks/stop.yml

@@ -0,0 +1 @@
+---

ansible/roles/ceph-rgw/tasks/upgrade.yml

@@ -0,0 +1 @@
+---

ansible/roles/haproxy/tasks/precheck.yml

@@ -200,6 +200,20 @@
     - haproxy_stat.find('blazar_api') == -1
     - haproxy_vip_prechecks
 
+- name: Checking free port for Ceph RadosGW HAProxy
+  wait_for:
+    host: "{{ kolla_internal_vip_address }}"
+    port: "{{ ceph_rgw_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - enable_ceph_rgw | bool
+    - enable_ceph_rgw_loadbalancer | bool
+    - inventory_hostname in groups['haproxy']
+    - haproxy_stat.find('radosgw') == -1
+    - haproxy_vip_prechecks
+
 - name: Checking free port for Cinder API HAProxy
   wait_for:
     host: "{{ kolla_internal_vip_address }}"

ansible/site.yml

@@ -20,6 +20,7 @@
         - enable_barbican_{{ enable_barbican | bool }}
         - enable_blazar_{{ enable_blazar | bool }}
         - enable_ceilometer_{{ enable_ceilometer | bool }}
+        - enable_ceph_rgw_{{ enable_ceph_rgw | bool }}
         - enable_chrony_{{ enable_chrony | bool }}
         - enable_cinder_{{ enable_cinder | bool }}
         - enable_cloudkitty_{{ enable_cloudkitty | bool }}
@@ -145,7 +146,12 @@
           tags: blazar
           when: enable_blazar | bool
         - include_role:
-            name: cinder
+            role: ceph-rgw
+            tasks_from: loadbalancer
+          tags: ceph-rgw
+          when: enable_ceph_rgw | bool
+        - include_role:
+            role: cinder
             tasks_from: loadbalancer
           tags: cinder
           when: enable_cinder | bool
@@ -637,6 +643,19 @@
         tags: swift,
         when: enable_swift | bool }
 
+- name: Apply role ceph-rgw
+  gather_facts: false
+  hosts:
+    # NOTE(mgoddard): This is only used to register Keystone services, and
+    # could run on any host running kolla-toolbox.
+    - kolla-toolbox
+    - '&enable_ceph_rgw_True'
+  serial: '{{ kolla_serial|default("0") }}'
+  roles:
+    - { role: ceph-rgw,
+        tags: ceph-rgw,
+        when: enable_ceph_rgw | bool }
+
 - name: Apply role glance
   gather_facts: false
   hosts:

doc/source/reference/storage/external-ceph-guide.rst

@@ -213,3 +213,68 @@ type ``default_share_type``, please see :doc:`Manila in Kolla <manila-guide>`.
 
 For more details on the CephFS Native driver, please see
 :manila-doc:`CephFS Native driver <admin/cephfs_driver.html>`.
+
+RadosGW
+-------
+
+As of the Wallaby 12.0.0 release, Kolla Ansible supports integration with Ceph
+RadosGW. This includes:
+
+* Registration of Swift-compatible endpoints in Keystone
+* Load balancing across RadosGW API servers using HAProxy
+
+See the `Ceph documentation
+<https://docs.ceph.com/en/latest/radosgw/keystone/>`__ for further information,
+including changes that must be applied to the Ceph cluster configuration.
+
+Enable Ceph RadosGW integration:
+
+.. code-block:: yaml
+
+   enable_ceph_rgw: true
+
+Keystone integration
+====================
+
+A Keystone user and endpoints are registered by default, however this may be
+avoided by setting ``enable_ceph_rgw_keystone`` to ``false``. If registration
+is enabled, the username is defined via ``ceph_rgw_keystone_user``, and this
+defaults to ``ceph_rgw``. The hostnames used by the endpoints default to
+``ceph_rgw_external_fqdn`` and ``ceph_rgw_internal_fqdn`` for the public and
+internal endpoints respectively. These default to ``kolla_external_fqdn`` and
+``kolla_internal_fqdn`` respectively. The port used by the endpoints is defined
+via ``ceph_rgw_port``, and defaults to 6780.
+
+By default RadosGW supports both Swift and S3 API, and it is not completely
+compatible with Swift API. The option ``ceph_rgw_compatibility`` can
+enable/disable complete RadosGW compatibility with Swift API.  After changing
+the value, run the ``kolla-ansible deploy`` command to enable.
+
+By default, the RadosGW endpoint URL does not include the project (account) ID.
+This prevents cross-project and public object access. This can be resolved by
+setting ``ceph_rgw_account_in_url`` to ``true``.
+
+Load balancing
+==============
+
+.. note::
+
+   Users of Ceph RadosGW can generate very high volumes of traffic. It is
+   advisable to use a separate load balancer for RadosGW for anything other
+   than small or lightly utilised RadosGW deployments.
+
+Load balancing is enabled by default, however this may be avoided by setting
+``enable_ceph_rgw_loadbalancer`` to ``false``. If using load balancing, the
+RadosGW hosts and ports must be configured. For example:
+
+.. code-block:: yaml
+
+   ceph_rgw_hosts:
+     - rgw-host-1:6780
+     - rgw-host-1:6780
+
+If using hostnames, these should be resolvable from the host running HAProxy.
+Alternatively IP addresses may be used.
+
+The HAProxy frontend port is defined via ``ceph_rgw_port``, and defaults to
+6780.

etc/kolla/globals.yml

@@ -273,6 +273,7 @@
 #enable_ceilometer_ipmi: "no"
 #enable_cells: "no"
 #enable_central_logging: "no"
+#enable_ceph_rgw: "no"
 #enable_chrony: "yes"
 #enable_cinder: "no"
 #enable_cinder_backup: "yes"

etc/kolla/passwords.yml

@@ -261,3 +261,8 @@ redis_master_password:
 ####################
 prometheus_mysql_exporter_database_password:
 prometheus_alertmanager_password:
+
+####################
+# Ceph RadosGW options
+####################
+ceph_rgw_keystone_password:

releasenotes/notes/ceph-rgw-062e0544a004f7b1.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds support for integration with Ceph RadosGW.

tests/templates/globals-default.j2

@@ -125,6 +125,11 @@ glance_backend_ceph: "yes"
 cinder_backend_ceph: "yes"
 nova_backend_ceph: "yes"
 ceph_nova_user: "cinder"
+enable_ceph_rgw: {{ not is_upgrade or previous_release != 'victoria' }}
+ceph_rgw_hosts:
+{% for host in hostvars %}
+  - {{ hostvars[host]['ansible_host'] }}:6780
+{% endfor %}
 {% endif %}
 
 {% if tls_enabled %}