From 5d94f74e80306ce8f8c5bf78efa8417e3fe7c6f0 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 7 May 2024 15:01:09 +0200 Subject: [PATCH 01/25] Create scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...N-mandatory-and-supported-IaaS-services.md | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md new file mode 100644 index 000000000..bd381c058 --- /dev/null +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -0,0 +1,80 @@ +--- +title: Mandatory and Supported IaaS Services +type: Standard +status: Draft +track: IaaS +--- + +## Introduction + +To be SCS-compliant a CSP has to fulfill all SCS-Standards. +Some of those standards are broad and consider ALL services on the IaaS-Layer. +There exist many services on that layer and they need to be limited to have a clear scope for the standards and the Compute Service Providers following them. +So this standard will provide lists for mandatory services that have to be present and supported services, which are considered in standards and may be tested or even implemented in the reference implementation. + +## Motivation + +There are many OpenStack APIs and their services that can be applied on IaaS-Level. +These services have differences in the quality of their implementation and liveness and some of them may be easily omitted when creating an IaaS-Deployment. +To fulfill all SCS-provided standards there are only some of these APIs required. +More but not all OpenStack services are tested or integrated in the reference implementation. +This document will give readers insight about how the SCS looks at all the OpenStack services. +If a cloud provides all mandatory and maybe some supported OpenStack APIs and implementation of their services it can be tested for SCS-compliance. +Any unsupported services will not be tested. + +## Mandatory OpenStack services + +The following OpenStack services MUST be present in SCS-compliant IaaS-Deployments: + +| OpenStack Service | description | +|-----|-----| +| **Cinder** | Block Storage service | +| **Glance** | Image service | +| **Keystone** | Identity service | +| **Neutron** | Networking service | +| **Nova** | Compute service | +| **Octavia** | Load-balancer service | +| **Placement** | Hardware Describing Service for Nova | +| **S3 API object storage** | No formal standard exists, many implementations: Swift, RadosGW, minio... | + +:::caution + +S3 API implementations may differ in certain offered features. +CSPs must publicly describe, which implementation they use in their deployment. +Users should always research whether a needed feature is supported in the offered implementation. + +::: + +## Supported OpenStack services + +The following services MAY be present in SCS-compliant IaaS-Deployment and they are considered in the SCS standards. +Most of these services (except Cloudkitty, Gnocchi and Masakari) have been integrated and tested by the SCS reference implementation: + +| OpenStack Service | description | +|-----|-----| +| **Barbican** | Key Manager service | +| **Cloudkitty** | Rating/Billing service | +| **Ceilometer** | Telemetry service | +| **Designate** | DNS service | +| **Gnocchi** | Time Series Database service | +| **Heat** | Orchestration service | +| **Horizon** | Dashboard | +| **Ironic** | Bare Metal provisioning service | +| **Manila** | Shared File Systems service | +| **Masakari** | Instances High Availability service | +| **Skyline** | Dashboard | + +## Unsupported OpenStack services + +All other OpenStack services that are not mentioned in the mandatory or supported lists are not tested for their integration and behavior by the SCS community. +Those services may be integrated into IaaS deployments by a CSP on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. +The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported in conjunction with an SCS-conformant infrastructure. + +## Related Documents + +[The OpenStack Services](https://www.openstack.org/software/) + +## Conformance Tests + +The presence of the mandatory OpenStack services (except the S3) will be tested in a test-script. +As S3 is a moving target, it may be integrated into the test, but will not let the Conformance test fail in general. From 7d8a5145d0b80338892ea6770f93e125b0e5b8c3 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 17 May 2024 08:54:59 +0200 Subject: [PATCH 02/25] Apply suggestions from code review Co-authored-by: Markus Hentsch <129268441+markus-hentsch@users.noreply.github.com> Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...N-mandatory-and-supported-IaaS-services.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index bd381c058..f3d7994b0 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -7,24 +7,25 @@ track: IaaS ## Introduction -To be SCS-compliant a CSP has to fulfill all SCS-Standards. +To be SCS-compliant a CSP has to fulfill all SCS standards. Some of those standards are broad and consider ALL services on the IaaS-Layer. -There exist many services on that layer and they need to be limited to have a clear scope for the standards and the Compute Service Providers following them. -So this standard will provide lists for mandatory services that have to be present and supported services, which are considered in standards and may be tested or even implemented in the reference implementation. +There exist many services on that layer and they need to be limited to have a clear scope for the standards and the Cloud Service Providers following them. +For this purpose, this standard will establish lists for mandatory services that have to be present in a SCS cloud as well as supported services, which are considered by some standards and may be tested or even implemented in the reference implementation but are optional in a sense that their omission will not violate SCS conformance. ## Motivation -There are many OpenStack APIs and their services that can be applied on IaaS-Level. -These services have differences in the quality of their implementation and liveness and some of them may be easily omitted when creating an IaaS-Deployment. +There are many OpenStack APIs and their corresponding services that can be deployed on the IaaS level. +These services have differences in the quality of their implementation and liveness and some of them may be easily omitted when creating an IaaS deployment. To fulfill all SCS-provided standards there are only some of these APIs required. -More but not all OpenStack services are tested or integrated in the reference implementation. -This document will give readers insight about how the SCS looks at all the OpenStack services. -If a cloud provides all mandatory and maybe some supported OpenStack APIs and implementation of their services it can be tested for SCS-compliance. +Some more but not all remaining OpenStack APIs are also supported additionally by the SCS project and may be part of its reference implementation. +This results in different levels of support for specific services. +This document will give readers insight about how the SCS classifies the OpenStack services accordingly. +If a cloud provides all mandatory and any number of supported OpenStack APIs, it can be tested for SCS-compliance. Any unsupported services will not be tested. ## Mandatory OpenStack services -The following OpenStack services MUST be present in SCS-compliant IaaS-Deployments: +The following OpenStack services MUST be present in SCS-compliant IaaS deployments: | OpenStack Service | description | |-----|-----| @@ -47,8 +48,7 @@ Users should always research whether a needed feature is supported in the offere ## Supported OpenStack services -The following services MAY be present in SCS-compliant IaaS-Deployment and they are considered in the SCS standards. -Most of these services (except Cloudkitty, Gnocchi and Masakari) have been integrated and tested by the SCS reference implementation: +The following services MAY be present in SCS-compliant IaaS deployment and are considered in the SCS standards. | OpenStack Service | description | |-----|-----| @@ -66,8 +66,8 @@ Most of these services (except Cloudkitty, Gnocchi and Masakari) have been integ ## Unsupported OpenStack services -All other OpenStack services that are not mentioned in the mandatory or supported lists are not tested for their integration and behavior by the SCS community. -Those services may be integrated into IaaS deployments by a CSP on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. +All other OpenStack services that are not mentioned in the mandatory or supported lists are not tested for their compatibility and behavior in SCS clouds by the SCS community. +Those services MAY be integrated into IaaS deployments by a CSP on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported in conjunction with an SCS-conformant infrastructure. ## Related Documents @@ -77,4 +77,4 @@ The SCS standard offers no guarantees for compatibility or reliability of servic ## Conformance Tests The presence of the mandatory OpenStack services (except the S3) will be tested in a test-script. -As S3 is a moving target, it may be integrated into the test, but will not let the Conformance test fail in general. +As the S3 interface is a moving target, it may be integrated into the test suite but the test result will not be taken into account to determine conformance. From bd523057d1d24ae18e96f9631834040e956e6b38 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 17 May 2024 08:57:29 +0200 Subject: [PATCH 03/25] Update scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index f3d7994b0..4a3842986 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -16,7 +16,7 @@ For this purpose, this standard will establish lists for mandatory services that There are many OpenStack APIs and their corresponding services that can be deployed on the IaaS level. These services have differences in the quality of their implementation and liveness and some of them may be easily omitted when creating an IaaS deployment. -To fulfill all SCS-provided standards there are only some of these APIs required. +To fulfill all SCS-provided standards only a subset of these APIs are required. Some more but not all remaining OpenStack APIs are also supported additionally by the SCS project and may be part of its reference implementation. This results in different levels of support for specific services. This document will give readers insight about how the SCS classifies the OpenStack services accordingly. From 1c3a6ad617245caf669ebae4c2a7994d5746302e Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 17 May 2024 14:50:06 +0200 Subject: [PATCH 04/25] Create test to check for mandatory services Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../mandatory-iaas-services.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 Tests/iaas/mandatory-services/mandatory-iaas-services.py diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py new file mode 100644 index 000000000..d71b13458 --- /dev/null +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -0,0 +1,94 @@ +"""Mandatory APIs checker + +This script retrieves the endpoint catalog from Keystone using the OpenStack +SDK and checks whether all mandatory APi endpoints, are present. +The script relies on an OpenStack SDK compatible clouds.yaml file for +authentication with Keystone. +As the s3 endpoint might differ, a missing one will only result in a warning. +""" + +import argparse +import getpass +import logging +import os + +import openstack + + +logger = logging.getLogger(__name__) +mandatory_services = ["compute", "identity", "image", "block-storage", + "network", "load-balancer", "s3", "placement"] + + +def connect(cloud_name: str) -> openstack.connection.Connection: + """Create a connection to an OpenStack cloud + :param string cloud_name: + The name of the configuration to load from clouds.yaml. + :returns: openstack.connnection.Connection + """ + return openstack.connect( + cloud=cloud_name, + ) + + +def check_presence_of_mandatory_services(cloud_name: str): + try: + connection = connect(cloud_name) + services = connection.service_catalog + except Exception as e: + print(str(e)) + raise Exception( + f"Connection to cloud '{cloud_name}' was not successfully. " + f"The Catalog endpoint could not be accessed. " + f"Please check your cloud connection and authorization." + ) + + for svc in services: + svc_type = svc['type'] + if svc_type in mandatory_services: + mandatory_services.remove(svc_type) + + if not mandatory_services: + # every mandatory service API had an endpoint + return 0 + else: + # if only s3 is not available, that might be named differently + if mandatory_services == ["s3"]: + logger.warning("No s3 endpoint found.") + return 0 + else: + # there were multiple mandatory APIs not found + logger.error(f"The following endpoints are missing: " + f"{mandatory_services}") + return len(mandatory_services) + + +def main(): + parser = argparse.ArgumentParser( + description="SCS Mandatory IaaS Service Checker") + parser.add_argument( + "--os-cloud", type=str, + help="Name of the cloud from clouds.yaml, alternative " + "to the OS_CLOUD environment variable" + ) + parser.add_argument( + "--debug", action="store_true", + help="Enable OpenStack SDK debug logging" + ) + args = parser.parse_args() + openstack.enable_logging(debug=args.debug) + + # parse cloud name for lookup in clouds.yaml + cloud = os.environ.get("OS_CLOUD", None) + if args.os_cloud: + cloud = args.os_cloud + assert cloud, ( + "You need to have the OS_CLOUD environment variable set to your cloud " + "name or pass it via --os-cloud" + ) + + return check_presence_of_mandatory_services(cloud) + + +if __name__ == "__main__": + main() From 99a2434048f44e60e60e2cf547607d0cec7f165b Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 17 May 2024 14:55:21 +0200 Subject: [PATCH 05/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Tests/iaas/mandatory-services/mandatory-iaas-services.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index d71b13458..7276b40c2 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -8,7 +8,6 @@ """ import argparse -import getpass import logging import os From 7ed642f61ae2d4c0574b4aae4172053dfebe42b0 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 28 May 2024 09:49:23 +0200 Subject: [PATCH 06/25] Apply suggestions from code review Co-authored-by: anjastrunk <119566837+anjastrunk@users.noreply.github.com> Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index 4a3842986..bb0a243d2 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -7,7 +7,7 @@ track: IaaS ## Introduction -To be SCS-compliant a CSP has to fulfill all SCS standards. +To be SCS-compliant a Cloud Service Provider (CSP) has to fulfill all SCS standards. Some of those standards are broad and consider ALL services on the IaaS-Layer. There exist many services on that layer and they need to be limited to have a clear scope for the standards and the Cloud Service Providers following them. For this purpose, this standard will establish lists for mandatory services that have to be present in a SCS cloud as well as supported services, which are considered by some standards and may be tested or even implemented in the reference implementation but are optional in a sense that their omission will not violate SCS conformance. @@ -67,7 +67,7 @@ The following services MAY be present in SCS-compliant IaaS deployment and are c ## Unsupported OpenStack services All other OpenStack services that are not mentioned in the mandatory or supported lists are not tested for their compatibility and behavior in SCS clouds by the SCS community. -Those services MAY be integrated into IaaS deployments by a CSP on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. +Those services MAY be integrated into IaaS deployments by a Cloud Service Provider on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported in conjunction with an SCS-conformant infrastructure. ## Related Documents From 1abf35beaef83a57bc27eff86e78bd233325f09c Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:22:36 +0200 Subject: [PATCH 07/25] Update scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index bb0a243d2..8e12268fd 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -66,7 +66,7 @@ The following services MAY be present in SCS-compliant IaaS deployment and are c ## Unsupported OpenStack services -All other OpenStack services that are not mentioned in the mandatory or supported lists are not tested for their compatibility and behavior in SCS clouds by the SCS community. +All other OpenStack services that are not mentioned in the mandatory or supported lists will not be tested for their compatibility and conformance in SCS clouds by the SCS community. Those services MAY be integrated into IaaS deployments by a Cloud Service Provider on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported in conjunction with an SCS-conformant infrastructure. From 2008cabd53b25ab22ce414ca1963ac1ccc45a13c Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:35:39 +0200 Subject: [PATCH 08/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../mandatory-iaas-services.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index 7276b40c2..2e091a4fa 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -16,7 +16,8 @@ logger = logging.getLogger(__name__) mandatory_services = ["compute", "identity", "image", "block-storage", - "network", "load-balancer", "s3", "placement"] + "network", "load-balancer", "placement"] +object_store_service = ["s3", "object-store"] def connect(cloud_name: str) -> openstack.connection.Connection: @@ -46,20 +47,22 @@ def check_presence_of_mandatory_services(cloud_name: str): svc_type = svc['type'] if svc_type in mandatory_services: mandatory_services.remove(svc_type) - + continue + if svc_type in object_store_service: + object_store_service.remove(svc_type) + + if len(object_store_service) == 2: + # neither s3 nor object-store is available, + # but might be named differently + logger.warning("No s3 or object-store endpoint found.") if not mandatory_services: # every mandatory service API had an endpoint return 0 else: - # if only s3 is not available, that might be named differently - if mandatory_services == ["s3"]: - logger.warning("No s3 endpoint found.") - return 0 - else: - # there were multiple mandatory APIs not found - logger.error(f"The following endpoints are missing: " - f"{mandatory_services}") - return len(mandatory_services) + # there were multiple mandatory APIs not found + logger.error(f"The following endpoints are missing: " + f"{mandatory_services}") + return len(mandatory_services) def main(): From d2e0a6fc25b8e57aaf3762700cb76f6c76e71001 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:05:19 +0200 Subject: [PATCH 09/25] Update scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...N-mandatory-and-supported-IaaS-services.md | 63 +++++++++---------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index 8e12268fd..262d7ed7c 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -8,9 +8,9 @@ track: IaaS ## Introduction To be SCS-compliant a Cloud Service Provider (CSP) has to fulfill all SCS standards. -Some of those standards are broad and consider ALL services on the IaaS-Layer. -There exist many services on that layer and they need to be limited to have a clear scope for the standards and the Cloud Service Providers following them. -For this purpose, this standard will establish lists for mandatory services that have to be present in a SCS cloud as well as supported services, which are considered by some standards and may be tested or even implemented in the reference implementation but are optional in a sense that their omission will not violate SCS conformance. +Some of those standards are broad and consider ALL APIs of ALL services on the IaaS-Layer. +There exist many services on that layer and for a first step they need to be limited to have a clear scope for the standards and the Cloud Service Providers following them. +For this purpose, this standard will establish lists for mandatory services that APIs have to be present in a SCS cloud as well as supported services, which APIS are considered by some standards and may even be tested for their integration but are optional in a sense that their omission will not violate SCS conformance. ## Motivation @@ -23,20 +23,19 @@ This document will give readers insight about how the SCS classifies the OpenSta If a cloud provides all mandatory and any number of supported OpenStack APIs, it can be tested for SCS-compliance. Any unsupported services will not be tested. -## Mandatory OpenStack services +## Mandatory IaaS APIs -The following OpenStack services MUST be present in SCS-compliant IaaS deployments: +The following IaaS APIs MUST be present in SCS-compliant IaaS deployments and could be implemented with the corresponding OpenStack services: -| OpenStack Service | description | -|-----|-----| -| **Cinder** | Block Storage service | -| **Glance** | Image service | -| **Keystone** | Identity service | -| **Neutron** | Networking service | -| **Nova** | Compute service | -| **Octavia** | Load-balancer service | -| **Placement** | Hardware Describing Service for Nova | -| **S3 API object storage** | No formal standard exists, many implementations: Swift, RadosGW, minio... | +| Mandatory API | corresponding OpenStack Service | description | +|-----|-----|-----| +| **block-storage** | Cinder | Block Storage service | +| **image** | Glance | Image service | +| **identity** | Keystone | Identity service | +| **network** | Neutron | Networking service | +| **compute** | Nova | Compute service | +| **load-balancer** | Octavia | Load-balancer service | +| **s3** or **object-store** | S3 API object storage | No formal standard exists, many implementations: Swift, RadosGW, minio... | :::caution @@ -46,27 +45,25 @@ Users should always research whether a needed feature is supported in the offere ::: -## Supported OpenStack services +## Supported IaaS APIs -The following services MAY be present in SCS-compliant IaaS deployment and are considered in the SCS standards. +The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. implemented thorugh the corresponding OpenStack services, and are considered in the SCS standards. -| OpenStack Service | description | -|-----|-----| -| **Barbican** | Key Manager service | -| **Cloudkitty** | Rating/Billing service | -| **Ceilometer** | Telemetry service | -| **Designate** | DNS service | -| **Gnocchi** | Time Series Database service | -| **Heat** | Orchestration service | -| **Horizon** | Dashboard | -| **Ironic** | Bare Metal provisioning service | -| **Manila** | Shared File Systems service | -| **Masakari** | Instances High Availability service | -| **Skyline** | Dashboard | +| Mandatory API | corresponding OpenStack Service | description | +|-----|-----|-----| +| **key-manager** | Barbican | Key Manager service | +| **billing** | Cloudkitty | Rating/Billing service | +| **telemetry** | Ceilomete | Telemetry service | +| **dns** | Designate | DNS service | +| **time-series-databse** | Gnocchi | Time Series Database service | +| **orchestration** | Heat | Orchestration service | +| **bare-metal** | Ironic | Bare Metal provisioning service | +| **shared-file-systems** | Manila | Shared File Systems service | +| **ha** | Masakari | Instances High Availability service | -## Unsupported OpenStack services +## Unsupported IaaS APIs -All other OpenStack services that are not mentioned in the mandatory or supported lists will not be tested for their compatibility and conformance in SCS clouds by the SCS community. +All other OpenStack services, which APIs are not mentioned in the mandatory or supported lists will not be tested for their compatibility and conformance in SCS clouds by the SCS community. Those services MAY be integrated into IaaS deployments by a Cloud Service Provider on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported in conjunction with an SCS-conformant infrastructure. @@ -76,5 +73,5 @@ The SCS standard offers no guarantees for compatibility or reliability of servic ## Conformance Tests -The presence of the mandatory OpenStack services (except the S3) will be tested in a test-script. +The presence of the mandatory OpenStack APIs (except the S3) will be tested in a test-script. As the S3 interface is a moving target, it may be integrated into the test suite but the test result will not be taken into account to determine conformance. From 9b97d9a6390b8e9319b85b6672cb00f25ac20f30 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:11:40 +0200 Subject: [PATCH 10/25] Apply suggestions from code review Co-authored-by: Sven Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...-XXXX-vN-mandatory-and-supported-IaaS-services.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index 262d7ed7c..4e48f4a68 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -8,9 +8,9 @@ track: IaaS ## Introduction To be SCS-compliant a Cloud Service Provider (CSP) has to fulfill all SCS standards. -Some of those standards are broad and consider ALL APIs of ALL services on the IaaS-Layer. +Some of those standards are broad and consider all APIs of all services on the IaaS-Layer. There exist many services on that layer and for a first step they need to be limited to have a clear scope for the standards and the Cloud Service Providers following them. -For this purpose, this standard will establish lists for mandatory services that APIs have to be present in a SCS cloud as well as supported services, which APIS are considered by some standards and may even be tested for their integration but are optional in a sense that their omission will not violate SCS conformance. +For this purpose, this standard will establish lists for mandatory services whose APIs have to be present in a SCS cloud as well as supported services, which APIs are considered by some standards and may even be tested for their integration but are optional in a sense that their omission will not violate SCS conformance. ## Motivation @@ -19,9 +19,9 @@ These services have differences in the quality of their implementation and liven To fulfill all SCS-provided standards only a subset of these APIs are required. Some more but not all remaining OpenStack APIs are also supported additionally by the SCS project and may be part of its reference implementation. This results in different levels of support for specific services. -This document will give readers insight about how the SCS classifies the OpenStack services accordingly. +This document will give readers insight about how the SCS classifies the OpenStack APIs accordingly. If a cloud provides all mandatory and any number of supported OpenStack APIs, it can be tested for SCS-compliance. -Any unsupported services will not be tested. +Any unsupported APIs will not be tested. ## Mandatory IaaS APIs @@ -63,9 +63,9 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im ## Unsupported IaaS APIs -All other OpenStack services, which APIs are not mentioned in the mandatory or supported lists will not be tested for their compatibility and conformance in SCS clouds by the SCS community. +All other OpenStack services, whose APIs are not mentioned in the mandatory or supported lists will not be tested for their compatibility and conformance in SCS clouds by the SCS community. Those services MAY be integrated into IaaS deployments by a Cloud Service Provider on their own responsibility but the SCS will not assume they are present and potential issues that occur during deployment or usage have to be handled by the CSP on their own accord. -The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported in conjunction with an SCS-conformant infrastructure. +The SCS standard offers no guarantees for compatibility or reliability of services categorized as unsupported. ## Related Documents From 8d5200c9471e396f445d94de6647af91cf3ab7c1 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:32:32 +0200 Subject: [PATCH 11/25] Update scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index 4e48f4a68..ae00cbe75 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -49,7 +49,7 @@ Users should always research whether a needed feature is supported in the offere The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. implemented thorugh the corresponding OpenStack services, and are considered in the SCS standards. -| Mandatory API | corresponding OpenStack Service | description | +| Supported API | corresponding OpenStack Service | description | |-----|-----|-----| | **key-manager** | Barbican | Key Manager service | | **billing** | Cloudkitty | Rating/Billing service | From 3b000139cc41c07ad23d948563964d25fb946f06 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:34:51 +0200 Subject: [PATCH 12/25] Apply suggestions from code review Co-authored-by: Markus Hentsch <129268441+markus-hentsch@users.noreply.github.com> Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index ae00cbe75..cd3f51cbc 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -53,7 +53,7 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im |-----|-----|-----| | **key-manager** | Barbican | Key Manager service | | **billing** | Cloudkitty | Rating/Billing service | -| **telemetry** | Ceilomete | Telemetry service | +| **telemetry** | Ceilometer | Telemetry service | | **dns** | Designate | DNS service | | **time-series-databse** | Gnocchi | Time Series Database service | | **orchestration** | Heat | Orchestration service | From bbbf13f5d654000ba5fe9339b19d728ce20e2117 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:37:14 +0200 Subject: [PATCH 13/25] Re-arranging APIs to alphabetical order Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...X-vN-mandatory-and-supported-IaaS-services.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index cd3f51cbc..e1a5f8f84 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -30,11 +30,11 @@ The following IaaS APIs MUST be present in SCS-compliant IaaS deployments and co | Mandatory API | corresponding OpenStack Service | description | |-----|-----|-----| | **block-storage** | Cinder | Block Storage service | -| **image** | Glance | Image service | -| **identity** | Keystone | Identity service | -| **network** | Neutron | Networking service | | **compute** | Nova | Compute service | +| **identity** | Keystone | Identity service | +| **image** | Glance | Image service | | **load-balancer** | Octavia | Load-balancer service | +| **network** | Neutron | Networking service | | **s3** or **object-store** | S3 API object storage | No formal standard exists, many implementations: Swift, RadosGW, minio... | :::caution @@ -51,15 +51,15 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im | Supported API | corresponding OpenStack Service | description | |-----|-----|-----| -| **key-manager** | Barbican | Key Manager service | +| **bare-metal** | Ironic | Bare Metal provisioning service | | **billing** | Cloudkitty | Rating/Billing service | -| **telemetry** | Ceilometer | Telemetry service | | **dns** | Designate | DNS service | -| **time-series-databse** | Gnocchi | Time Series Database service | +| **ha** | Masakari | Instances High Availability service | +| **key-manager** | Barbican | Key Manager service | | **orchestration** | Heat | Orchestration service | -| **bare-metal** | Ironic | Bare Metal provisioning service | | **shared-file-systems** | Manila | Shared File Systems service | -| **ha** | Masakari | Instances High Availability service | +| **telemetry** | Ceilometer | Telemetry service | +| **time-series-databse** | Gnocchi | Time Series Database service | ## Unsupported IaaS APIs From 9ca339cb3f36c2e7c6d0c3b464111c1084155748 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:20:24 +0200 Subject: [PATCH 14/25] Name a specific standard (role standard) that requires knowledge about mandatory and supported services. Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index e1a5f8f84..d92b01baa 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -8,7 +8,7 @@ track: IaaS ## Introduction To be SCS-compliant a Cloud Service Provider (CSP) has to fulfill all SCS standards. -Some of those standards are broad and consider all APIs of all services on the IaaS-Layer. +Some of those standards are broad and consider all APIs of all services on the IaaS-Layer like the consideration of a [role standard](https://github.com/SovereignCloudStack/issues/issues/396). There exist many services on that layer and for a first step they need to be limited to have a clear scope for the standards and the Cloud Service Providers following them. For this purpose, this standard will establish lists for mandatory services whose APIs have to be present in a SCS cloud as well as supported services, which APIs are considered by some standards and may even be tested for their integration but are optional in a sense that their omission will not violate SCS conformance. From 7858db1632740f644be22a8fe0e7bc7a5df2c5a2 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:13:29 +0200 Subject: [PATCH 15/25] Update scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index d92b01baa..1afadb392 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -35,7 +35,7 @@ The following IaaS APIs MUST be present in SCS-compliant IaaS deployments and co | **image** | Glance | Image service | | **load-balancer** | Octavia | Load-balancer service | | **network** | Neutron | Networking service | -| **s3** or **object-store** | S3 API object storage | No formal standard exists, many implementations: Swift, RadosGW, minio... | +| **s3** | S3 API object storage | Object Storage service | :::caution @@ -56,6 +56,7 @@ The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. im | **dns** | Designate | DNS service | | **ha** | Masakari | Instances High Availability service | | **key-manager** | Barbican | Key Manager service | +| **object-store** | Swift | Object Store with different possible backends | | **orchestration** | Heat | Orchestration service | | **shared-file-systems** | Manila | Shared File Systems service | | **telemetry** | Ceilometer | Telemetry service | From a07b7187516f685332bd1a384b73b191d4ed2a7b Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:18:30 +0200 Subject: [PATCH 16/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../mandatory-services/mandatory-iaas-services.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index 2e091a4fa..0d8b8102e 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -16,8 +16,9 @@ logger = logging.getLogger(__name__) mandatory_services = ["compute", "identity", "image", "block-storage", - "network", "load-balancer", "placement"] -object_store_service = ["s3", "object-store"] + "network", "load-balancer", "placement", + "object-store"] +# object_store_service = ["s3", "object-store"] def connect(cloud_name: str) -> openstack.connection.Connection: @@ -48,13 +49,14 @@ def check_presence_of_mandatory_services(cloud_name: str): if svc_type in mandatory_services: mandatory_services.remove(svc_type) continue - if svc_type in object_store_service: - object_store_service.remove(svc_type) + # the follwing code was used for mulitple object-store names + # if svc_type in object_store_service: + # object_store_service.remove(svc_type) - if len(object_store_service) == 2: + # if len(object_store_service) == 2: # neither s3 nor object-store is available, # but might be named differently - logger.warning("No s3 or object-store endpoint found.") + # logger.warning("No s3 or object-store endpoint found.") if not mandatory_services: # every mandatory service API had an endpoint return 0 From aebc1c29213077111bdeb3c3c4c2585c40b5125c Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:49:15 +0200 Subject: [PATCH 17/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../mandatory-iaas-services.py | 187 ++++++++++++++++-- 1 file changed, 170 insertions(+), 17 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index 0d8b8102e..50ac900a8 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -1,5 +1,4 @@ """Mandatory APIs checker - This script retrieves the endpoint catalog from Keystone using the OpenStack SDK and checks whether all mandatory APi endpoints, are present. The script relies on an OpenStack SDK compatible clouds.yaml file for @@ -8,17 +7,23 @@ """ import argparse +import boto3 +from collections import Counter import logging import os +import re +import sys +import uuid import openstack +TESTCONTNAME = "scs-test-container" + logger = logging.getLogger(__name__) -mandatory_services = ["compute", "identity", "image", "block-storage", - "network", "load-balancer", "placement", - "object-store"] -# object_store_service = ["s3", "object-store"] +mandatory_services = ["compute", "identity", "image", "network", + "load-balancer", "placement", "object-store"] +block_storage_service = ["volume", "volumev3", "block-storage"] def connect(cloud_name: str) -> openstack.connection.Connection: @@ -49,24 +54,169 @@ def check_presence_of_mandatory_services(cloud_name: str): if svc_type in mandatory_services: mandatory_services.remove(svc_type) continue - # the follwing code was used for mulitple object-store names - # if svc_type in object_store_service: - # object_store_service.remove(svc_type) - - # if len(object_store_service) == 2: - # neither s3 nor object-store is available, - # but might be named differently - # logger.warning("No s3 or object-store endpoint found.") + if svc_type in block_storage_service: + block_storage_service.remove(svc_type) + + bs_service_not_present = 0 + if len(block_storage_service) == 3: + # neither block-storage nor volume nor volumev3 is present + # we must assume, that there is no volume service + logger.error("FAIL: No block-storage (volume) endpoint found.") + mandatory_services.append(block_storage_service[0]) + bs_service_not_present = 1 if not mandatory_services: # every mandatory service API had an endpoint - return 0 + return 0 + bs_service_not_present else: # there were multiple mandatory APIs not found - logger.error(f"The following endpoints are missing: " + logger.error(f"FAIL: The following endpoints are missing: " f"{mandatory_services}") - return len(mandatory_services) + return len(mandatory_services) + bs_service_not_present + + +def list_containers(conn): + "Gets a list of buckets" + return [cont.name for cont in conn.object_store.containers()] + + +def create_container(conn, name): + "Creates a test container" + conn.object_store.create_container(name) + return list_containers(conn) + + +def del_container(conn, name): + "Deletes a test container" + conn.object_store.delete(name) + # return list_containers(conn) + + +def s3_conn(creds, conn=None): + "Return an s3 client conn" + vrfy = True + cacert = conn.config.config.get("cacert") + # TODO: Handle self-signed certs (from ca_cert in openstack config) + if cacert: + print("WARNING: Trust all Certificates in S3, " + f"OpenStack uses {cacert}", file=sys.stderr) + vrfy = False + return boto3.resource('s3', aws_access_key_id=creds["AKI"], + aws_secret_access_key=creds["SAK"], + endpoint_url=creds["HOST"], + verify=vrfy) + + +def list_s3_buckets(s3): + "Get a list of s3 buckets" + return [buck.name for buck in s3.buckets.all()] + + +def create_bucket(s3, name): + "Create an s3 bucket" + # bucket = s3.Bucket(name) + # bucket.create() + s3.create_bucket(Bucket=name) + return list_s3_buckets(s3) + + +def del_bucket(s3, name): + "Delete an s3 bucket" + buck = s3.Bucket(name=name) + buck.delete() + # s3.delete_bucket(Bucket=name) + + +def s3_from_env(creds, fieldnm, env, prefix=""): + "Set creds[fieldnm] to os.environ[env] if set" + if env in os.environ: + creds[fieldnm] = prefix + os.environ[env] + if fieldnm not in creds: + print(f"WARNING: s3_creds[{fieldnm}] not set", file=sys.stderr) + + +def s3_from_ostack(creds, conn, endpoint): + "Set creds from openstack swift/keystone" + rgx = re.compile(r"^(https*://[^/]*)/") + match = rgx.match(endpoint) + if match: + creds["HOST"] = match.group(1) + # Use first ec2 cred if one exists + ec2_creds = [cred for cred in conn.identity.credentials() + if cred.type == "ec2"] + if len(ec2_creds): + # FIXME: Assume cloud is not evil + ec2_dict = eval(ec2_creds[0].blob, {"null": None}) + creds["AKI"] = ec2_dict["access"] + creds["SAK"] = ec2_dict["secret"] + return + # Generate keyid and secret + aki = uuid.uuid4().hex + sak = uuid.uuid4().hex + blob = f'{{"access": "{aki}", "secret": "{sak}"}}' + try: + conn.identity.create_credential(type="ec2", blob=blob, + user_id=conn.current_user_id, + project_id=conn.current_project_id) + creds["AKI"] = aki + creds["SAK"] = sak + except BaseException as exc: + print(f"WARNING: ec2 creds creation failed: {exc!s}", file=sys.stderr) + # pass +def check_for_s3_and_swift(cloud_name: str): + try: + connection = connect(cloud_name) + connection.authorize() + except Exception as e: + print(str(e)) + raise Exception( + f"Connection to cloud '{cloud_name}' was not successfully. " + f"The Catalog endpoint could not be accessed. " + f"Please check your cloud connection and authorization." + ) + s3_creds = {} + try: + endpoint = connection.object_store.get_endpoint() + except Exception as e: + logger.error( + f"FAIL: No object store endpoint found. No testing for " + f"the s3 service possible." + ) + return 1 + # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) + s3_from_ostack(s3_creds, connection, endpoint) + # Overrides (var names are from libs3, in case you wonder) + s3_from_env(s3_creds, "HOST", "S3_HOSTNAME", "https://") + s3_from_env(s3_creds, "AKI", "S3_ACCESS_KEY_ID") + s3_from_env(s3_creds, "SAK", "S3_SECRET_ACCESS_KEY") + + s3 = s3_conn(s3_creds, connection) + s3_buckets = list_s3_buckets(s3) + if not s3_buckets: + s3_buckets = create_bucket(s3, TESTCONTNAME) + assert s3_buckets + + # If we got till here, s3 is working, now swift + swift_containers = list_containers(connection) + # if not swift_containers: + # swift_containers = create_container(connection, TESTCONTNAME) + result = 0 + if Counter(s3_buckets) != Counter(swift_containers): + print("WARNING: S3 buckets and Swift Containers differ:\n" + f"S3: {sorted(s3_buckets)}\nSW: {sorted(swift_containers)}") + result = 1 + else: + print("SUCCESS: S3 and Swift exist and agree") + # Clean up + # FIXME: Cleanup created EC2 credential + # if swift_containers == [TESTCONTNAME]: + # del_container(connection, TESTCONTNAME) + # Cleanup created S3 bucket + if s3_buckets == [TESTCONTNAME]: + del_bucket(s3, TESTCONTNAME) + return result + def main(): parser = argparse.ArgumentParser( description="SCS Mandatory IaaS Service Checker") @@ -91,7 +241,10 @@ def main(): "name or pass it via --os-cloud" ) - return check_presence_of_mandatory_services(cloud) + result = check_presence_of_mandatory_services(cloud) + result = result + check_for_s3_and_swift(cloud) + + return result if __name__ == "__main__": From 12b88a984970cd592f506379f99640bf52b9ba3d Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:02:16 +0200 Subject: [PATCH 18/25] Update scs-XXXX-vN-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../scs-XXXX-vN-mandatory-and-supported-IaaS-services.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md index 1afadb392..274738be9 100644 --- a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md +++ b/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md @@ -45,6 +45,10 @@ Users should always research whether a needed feature is supported in the offere ::: +The endpoints of services MUST be findable through the `catalog list` of the identity API[^1]. + +[^1]: [Integrate into the service catalog of Keystone](https://docs.openstack.org/keystone/latest/contributor/service-catalog.html) + ## Supported IaaS APIs The following IaaS APIs MAY be present in SCS-compliant IaaS deployment, e.g. implemented thorugh the corresponding OpenStack services, and are considered in the SCS standards. @@ -74,5 +78,5 @@ The SCS standard offers no guarantees for compatibility or reliability of servic ## Conformance Tests -The presence of the mandatory OpenStack APIs (except the S3) will be tested in a test-script. -As the S3 interface is a moving target, it may be integrated into the test suite but the test result will not be taken into account to determine conformance. +The presence of the mandatory OpenStack APIs will be tested in [this test-script](https://github.com/SovereignCloudStack/standards/blob/mandatory-and-supported-IaaS-services/Tests/iaas/mandatory-services/mandatory-iaas-services.py). +The test will further check, whether the object store endpoint is compatible to s3. From 2795cbbd4e1218c3906b58e9fa7c787524989ce0 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:07:39 +0200 Subject: [PATCH 19/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Tests/iaas/mandatory-services/mandatory-iaas-services.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index 50ac900a8..df8954455 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -181,8 +181,8 @@ def check_for_s3_and_swift(cloud_name: str): except Exception as e: logger.error( f"FAIL: No object store endpoint found. No testing for " - f"the s3 service possible." - ) + f"the s3 service possible. Details: %s", e + ) return 1 # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) s3_from_ostack(s3_creds, connection, endpoint) @@ -217,6 +217,7 @@ def check_for_s3_and_swift(cloud_name: str): del_bucket(s3, TESTCONTNAME) return result + def main(): parser = argparse.ArgumentParser( description="SCS Mandatory IaaS Service Checker") From 673f0e56f711b17cdfc64b4cac47e807167cb19b Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:10:44 +0200 Subject: [PATCH 20/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Tests/iaas/mandatory-services/mandatory-iaas-services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index df8954455..83e11ad79 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -181,7 +181,7 @@ def check_for_s3_and_swift(cloud_name: str): except Exception as e: logger.error( f"FAIL: No object store endpoint found. No testing for " - f"the s3 service possible. Details: %s", e + f"the s3 service possible in '{cloud_name}'. Details: %s", e ) return 1 # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) From 9e4fb388e61de177d8878da162f7b719ee583e25 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:45:13 +0200 Subject: [PATCH 21/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../mandatory-iaas-services.py | 71 +++++++++++++++---- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index 83e11ad79..a70c467bd 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -37,7 +37,7 @@ def connect(cloud_name: str) -> openstack.connection.Connection: ) -def check_presence_of_mandatory_services(cloud_name: str): +def check_presence_of_mandatory_services(cloud_name: str, s3_credentials=None): try: connection = connect(cloud_name) services = connection.service_catalog @@ -49,6 +49,8 @@ def check_presence_of_mandatory_services(cloud_name: str): f"Please check your cloud connection and authorization." ) + if s3_credentials: + mandatory_services.remove("object-store") for svc in services: svc_type = svc['type'] if svc_type in mandatory_services: @@ -94,12 +96,13 @@ def del_container(conn, name): def s3_conn(creds, conn=None): "Return an s3 client conn" vrfy = True - cacert = conn.config.config.get("cacert") - # TODO: Handle self-signed certs (from ca_cert in openstack config) - if cacert: - print("WARNING: Trust all Certificates in S3, " - f"OpenStack uses {cacert}", file=sys.stderr) - vrfy = False + if conn: + cacert = conn.config.config.get("cacert") + # TODO: Handle self-signed certs (from ca_cert in openstack config) + if cacert: + print("WARNING: Trust all Certificates in S3, " + f"OpenStack uses {cacert}", file=sys.stderr) + vrfy = False return boto3.resource('s3', aws_access_key_id=creds["AKI"], aws_secret_access_key=creds["SAK"], endpoint_url=creds["HOST"], @@ -164,7 +167,26 @@ def s3_from_ostack(creds, conn, endpoint): # pass -def check_for_s3_and_swift(cloud_name: str): +def check_for_s3_and_swift(cloud_name: str, s3_credentials=None): + # If we get credentials we assume, that there is no Swift and only test s3 + if s3_credentials: + try: + s3 = s3_conn(s3_credentials) + except Exception as e: + print(str(e)) + logger.error("FAIL: Connection to s3 failed.") + return 1 + s3_buckets = list_s3_buckets(s3) + if not s3_buckets: + s3_buckets = create_bucket(s3, TESTCONTNAME) + assert s3_buckets + if s3_buckets == [TESTCONTNAME]: + del_bucket(s3, TESTCONTNAME) + # everything worked, and we don't need to test for Swift: + print("SUCCESS: S3 exists") + return 0 + # there were no credentials given, so we assume s3 is accessable via + # the service catalog and Swift might exist too try: connection = connect(cloud_name) connection.authorize() @@ -181,8 +203,8 @@ def check_for_s3_and_swift(cloud_name: str): except Exception as e: logger.error( f"FAIL: No object store endpoint found. No testing for " - f"the s3 service possible in '{cloud_name}'. Details: %s", e - ) + f"the s3 service possible. Details: %s", e + ) return 1 # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) s3_from_ostack(s3_creds, connection, endpoint) @@ -217,7 +239,6 @@ def check_for_s3_and_swift(cloud_name: str): del_bucket(s3, TESTCONTNAME) return result - def main(): parser = argparse.ArgumentParser( description="SCS Mandatory IaaS Service Checker") @@ -226,6 +247,18 @@ def main(): help="Name of the cloud from clouds.yaml, alternative " "to the OS_CLOUD environment variable" ) + parser.add_argument( + "--s3-endpoint", type=str, + help="URL to the s3 service." + ) + parser.add_argument( + "--s3-access", type=str, + help="Access Key to connect to the s3 service." + ) + parser.add_argument( + "--s3-access-secret", type=str, + help="Access secret to connect to the s3 service." + ) parser.add_argument( "--debug", action="store_true", help="Enable OpenStack SDK debug logging" @@ -242,8 +275,20 @@ def main(): "name or pass it via --os-cloud" ) - result = check_presence_of_mandatory_services(cloud) - result = result + check_for_s3_and_swift(cloud) + s3_credentials = None + if args.s3_endpoint: + if (not args.s3_access) or (not args.s3_access_secret): + print("WARNING: test for external s3 needs access key and access secret.") + s3_credentials = { + "AKI": args.s3_access, + "SAK": args.s3_access_secret, + "HOST": args.s3_endpoint + } + elif args.s3_access or args.s3_access_secret: + print("WARNING: access to s3 was given, but no endpoint provided.") + + result = check_presence_of_mandatory_services(cloud, s3_credentials) + result = result + check_for_s3_and_swift(cloud, s3_credentials) return result From c1c4009d824f4ad90ea5270806d82834d7fa3b61 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:44:03 +0200 Subject: [PATCH 22/25] Update mandatory-iaas-services.py Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Tests/iaas/mandatory-services/mandatory-iaas-services.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index a70c467bd..652271114 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -202,9 +202,10 @@ def check_for_s3_and_swift(cloud_name: str, s3_credentials=None): endpoint = connection.object_store.get_endpoint() except Exception as e: logger.error( - f"FAIL: No object store endpoint found. No testing for " - f"the s3 service possible. Details: %s", e - ) + f"FAIL: No object store endpoint found in cloud " + f"'{cloud_name}'. No testing for the s3 service possible. " + f"Details: %s", e + ) return 1 # Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone) s3_from_ostack(s3_creds, connection, endpoint) @@ -239,6 +240,7 @@ def check_for_s3_and_swift(cloud_name: str, s3_credentials=None): del_bucket(s3, TESTCONTNAME) return result + def main(): parser = argparse.ArgumentParser( description="SCS Mandatory IaaS Service Checker") From 30a5eb0eacb55c30c9ce508c32852b0211a0cab0 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:31:59 +0200 Subject: [PATCH 23/25] Create README.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- Tests/iaas/mandatory-services/README.md | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 Tests/iaas/mandatory-services/README.md diff --git a/Tests/iaas/mandatory-services/README.md b/Tests/iaas/mandatory-services/README.md new file mode 100644 index 000000000..33a66d7f4 --- /dev/null +++ b/Tests/iaas/mandatory-services/README.md @@ -0,0 +1,66 @@ +# Mandatory IaaS Service APIs Test Suite + +## Test Environment Setup + +### Test Execution Environment + +> **NOTE:** The test execution procedure does not require cloud admin rights. + +To execute the test suite a valid cloud configuration for the OpenStack SDK in the shape of "`clouds.yaml`" is mandatory[^1]. +**The file is expected to be located in the current working directory where the test script is executed unless configured otherwise.** + +[^1]: [OpenStack Documentation: Configuring OpenStack SDK Applications](https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html) + +The test execution environment can be located on any system outside of the cloud infrastructure that has OpenStack API access. +Make sure that the API access is configured properly in "`clouds.yaml`". + +It is recommended to use a Python virtual environment[^2]. +Next, install the OpenStack SDK and boto3 required by the test suite: + +```bash +pip3 install openstacksdk +pip3 install boto3 +``` + +Within this environment execute the test suite. + +[^2]: [Python 3 Documentation: Virtual Environments and Packages](https://docs.python.org/3/tutorial/venv.html) + +## Test Execution + +The test suite is executed as follows: + +```bash +python3 mandatory-iaas-services.py --os-cloud mycloud +``` + +As an alternative to "`--os-cloud`", the "`OS_CLOUD`" environment variable may be specified instead. +The parameter is used to look up the correct cloud configuration in "`clouds.yaml`". +For the example command above, this file should contain a `clouds.mycloud` section like this: + +```yaml +--- +clouds: + mycloud: + auth: + auth_url: ... + ... + ... +``` + +If the deployment uses s3 only and does not have the object store endpoint specified in the service catalog, the "`--s3-endpoint`" flag may be used to specify the s3 endpoint. +In that case the "`--s3-access`" and "`--s3-access-secret`" flags must also be set, to give all necessery credentials to the test suite: + +```bash +python3 mandatory-iaas-services3.py --os-cloud mycloud2 --s3-endpoint "http://s3-endpoint:9000" --s3-access test-user --s3-access-secret test-user-secret +``` + +For any further options consult the output of "`python3 volume-backup-tester.py --help`". + +### Script Behavior & Test Results + +If all tests pass, the script will return with an exit code of `0`. + +If any test fails, the script will halt, print the exact error to `stderr` and return with a non-zero exit code. + +There is no cleanup done by this test as it mainly only inspect the service catalog and only for the object store creates a bucket, which is then promptly deleted. From c2a1b8b4e7973d328574ce35bbcf55ea2cbea000 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:45:33 +0100 Subject: [PATCH 24/25] change aki to ak and sak to sk Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- .../mandatory-iaas-services.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Tests/iaas/mandatory-services/mandatory-iaas-services.py b/Tests/iaas/mandatory-services/mandatory-iaas-services.py index 652271114..ab5cc0a2f 100644 --- a/Tests/iaas/mandatory-services/mandatory-iaas-services.py +++ b/Tests/iaas/mandatory-services/mandatory-iaas-services.py @@ -103,8 +103,8 @@ def s3_conn(creds, conn=None): print("WARNING: Trust all Certificates in S3, " f"OpenStack uses {cacert}", file=sys.stderr) vrfy = False - return boto3.resource('s3', aws_access_key_id=creds["AKI"], - aws_secret_access_key=creds["SAK"], + return boto3.resource('s3', aws_access_key_id=creds["AK"], + aws_secret_access_key=creds["SK"], endpoint_url=creds["HOST"], verify=vrfy) @@ -149,19 +149,19 @@ def s3_from_ostack(creds, conn, endpoint): if len(ec2_creds): # FIXME: Assume cloud is not evil ec2_dict = eval(ec2_creds[0].blob, {"null": None}) - creds["AKI"] = ec2_dict["access"] - creds["SAK"] = ec2_dict["secret"] + creds["AK"] = ec2_dict["access"] + creds["SK"] = ec2_dict["secret"] return # Generate keyid and secret - aki = uuid.uuid4().hex - sak = uuid.uuid4().hex - blob = f'{{"access": "{aki}", "secret": "{sak}"}}' + ak = uuid.uuid4().hex + sk = uuid.uuid4().hex + blob = f'{{"access": "{ak}", "secret": "{sk}"}}' try: conn.identity.create_credential(type="ec2", blob=blob, user_id=conn.current_user_id, project_id=conn.current_project_id) - creds["AKI"] = aki - creds["SAK"] = sak + creds["AK"] = ak + creds["SK"] = sk except BaseException as exc: print(f"WARNING: ec2 creds creation failed: {exc!s}", file=sys.stderr) # pass @@ -211,8 +211,8 @@ def check_for_s3_and_swift(cloud_name: str, s3_credentials=None): s3_from_ostack(s3_creds, connection, endpoint) # Overrides (var names are from libs3, in case you wonder) s3_from_env(s3_creds, "HOST", "S3_HOSTNAME", "https://") - s3_from_env(s3_creds, "AKI", "S3_ACCESS_KEY_ID") - s3_from_env(s3_creds, "SAK", "S3_SECRET_ACCESS_KEY") + s3_from_env(s3_creds, "AK", "S3_ACCESS_KEY_ID") + s3_from_env(s3_creds, "SK", "S3_SECRET_ACCESS_KEY") s3 = s3_conn(s3_creds, connection) s3_buckets = list_s3_buckets(s3) @@ -282,8 +282,8 @@ def main(): if (not args.s3_access) or (not args.s3_access_secret): print("WARNING: test for external s3 needs access key and access secret.") s3_credentials = { - "AKI": args.s3_access, - "SAK": args.s3_access_secret, + "AK": args.s3_access, + "SK": args.s3_access_secret, "HOST": args.s3_endpoint } elif args.s3_access or args.s3_access_secret: From 68df301c8990af1c0b3d945a78fafb8c692e8906 Mon Sep 17 00:00:00 2001 From: josephineSei <128813814+josephineSei@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:48:56 +0100 Subject: [PATCH 25/25] Rename scs-XXXX-vN-mandatory-and-supported-IaaS-services.md to scs-0123-v1-mandatory-and-supported-IaaS-services.md Signed-off-by: josephineSei <128813814+josephineSei@users.noreply.github.com> --- ...es.md => scs-0123-v1-mandatory-and-supported-IaaS-services.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Standards/{scs-XXXX-vN-mandatory-and-supported-IaaS-services.md => scs-0123-v1-mandatory-and-supported-IaaS-services.md} (100%) diff --git a/Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md b/Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md similarity index 100% rename from Standards/scs-XXXX-vN-mandatory-and-supported-IaaS-services.md rename to Standards/scs-0123-v1-mandatory-and-supported-IaaS-services.md