From 676465c53bacda4e8729bb87879105c0fba2dcfc Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Fri, 3 Jan 2025 12:19:45 +0000 Subject: [PATCH 1/3] Drop resources from HV props which return none These resources all return None for HVs, as Yoga+ expects us to use a resource object instead. Remove these since they will not work again, and any downstream usage will fail loudly when they try to import the dead enum. This will require a major version bump which will come in the next commit adding the placement replacement. --- .../enums/props/hypervisor_properties.py | 59 ------- openstackquery/mappings/hypervisor_mapping.py | 26 +-- openstackquery/runners/hypervisor_runner.py | 51 +----- .../enums/props/test_hypervisor_properties.py | 157 ------------------ tests/mappings/test_hypervisor_mapping.py | 26 --- tests/runners/test_hypervisor_runner.py | 35 ---- 6 files changed, 3 insertions(+), 351 deletions(-) diff --git a/openstackquery/enums/props/hypervisor_properties.py b/openstackquery/enums/props/hypervisor_properties.py index 8c1ac0f..4073cd0 100644 --- a/openstackquery/enums/props/hypervisor_properties.py +++ b/openstackquery/enums/props/hypervisor_properties.py @@ -12,21 +12,11 @@ class HypervisorProperties(PropEnum): An enum class for all hypervisor properties """ - # HYPERVISOR_CURRENT_WORKLOAD = auto() - HYPERVISOR_DISK_FREE = auto() - HYPERVISOR_DISK_SIZE = auto() - HYPERVISOR_DISK_USED = auto() HYPERVISOR_ID = auto() HYPERVISOR_IP = auto() - HYPERVISOR_MEMORY_FREE = auto() - HYPERVISOR_MEMORY_SIZE = auto() - HYPERVISOR_MEMORY_USED = auto() HYPERVISOR_NAME = auto() - # HYPERVISOR_SERVER_COUNT = auto() # Deprecated, use server query HYPERVISOR_STATE = auto() HYPERVISOR_STATUS = auto() - HYPERVISOR_VCPUS = auto() - HYPERVISOR_VCPUS_USED = auto() HYPERVISOR_DISABLED_REASON = auto() HYPERVISOR_UPTIME_DAYS = auto() @@ -36,33 +26,11 @@ def _get_aliases() -> Dict: A method that returns all valid string alias mappings """ return { - # HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD: [ - # "current_workload", - # "workload", - # ], - HypervisorProperties.HYPERVISOR_DISK_FREE: [ - "local_disk_free", - "free_disk_gb", - ], - HypervisorProperties.HYPERVISOR_DISK_SIZE: ["local_disk_size", "local_gb"], - HypervisorProperties.HYPERVISOR_DISK_USED: [ - "local_disk_used", - "local_gb_used", - ], HypervisorProperties.HYPERVISOR_ID: ["id", "uuid", "host_id"], HypervisorProperties.HYPERVISOR_IP: ["ip", "host_ip"], - HypervisorProperties.HYPERVISOR_MEMORY_FREE: ["memory_free", "free_ram_mb"], - HypervisorProperties.HYPERVISOR_MEMORY_SIZE: ["memory_size", "memory_mb"], - HypervisorProperties.HYPERVISOR_MEMORY_USED: [ - "memory_used", - "memory_mb_used", - ], HypervisorProperties.HYPERVISOR_NAME: ["name", "host_name"], - # HypervisorProperties.HYPERVISOR_SERVER_COUNT: ["running_vms"], HypervisorProperties.HYPERVISOR_STATE: ["state"], HypervisorProperties.HYPERVISOR_STATUS: ["status"], - HypervisorProperties.HYPERVISOR_VCPUS: ["vcpus"], - HypervisorProperties.HYPERVISOR_VCPUS_USED: ["vcpus_used"], HypervisorProperties.HYPERVISOR_DISABLED_REASON: ["disabled_reason"], HypervisorProperties.HYPERVISOR_UPTIME_DAYS: ["uptime"], } @@ -76,39 +44,12 @@ def get_prop_mapping(prop) -> Optional[PropFunc]: :param prop: A HypervisorProperty Enum for which a function may exist for """ mapping = { - # HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD: lambda a: a[ - # "current_workload" - # ], - HypervisorProperties.HYPERVISOR_DISK_FREE: lambda a: a.resources["DISK_GB"][ - "free" - ], - HypervisorProperties.HYPERVISOR_DISK_SIZE: lambda a: a.resources["DISK_GB"][ - "total" - ], - HypervisorProperties.HYPERVISOR_DISK_USED: lambda a: a.resources["DISK_GB"][ - "usage" - ], HypervisorProperties.HYPERVISOR_ID: lambda a: a["id"], HypervisorProperties.HYPERVISOR_IP: lambda a: a["host_ip"], - HypervisorProperties.HYPERVISOR_MEMORY_FREE: lambda a: a.resources[ - "MEMORY_MB" - ]["free"], - HypervisorProperties.HYPERVISOR_MEMORY_SIZE: lambda a: a.resources[ - "MEMORY_MB" - ]["total"], - HypervisorProperties.HYPERVISOR_MEMORY_USED: lambda a: a.resources[ - "MEMORY_MB" - ]["usage"], HypervisorProperties.HYPERVISOR_NAME: lambda a: a["name"], # HypervisorProperties.HYPERVISOR_SERVER_COUNT: lambda a: a["runnning_vms"], HypervisorProperties.HYPERVISOR_STATE: lambda a: a["state"], HypervisorProperties.HYPERVISOR_STATUS: lambda a: a["status"], - HypervisorProperties.HYPERVISOR_VCPUS: lambda a: a.resources["VCPU"][ - "total" - ], - HypervisorProperties.HYPERVISOR_VCPUS_USED: lambda a: a.resources["VCPU"][ - "usage" - ], HypervisorProperties.HYPERVISOR_DISABLED_REASON: lambda a: a["service"][ "disabled_reason" ], diff --git a/openstackquery/mappings/hypervisor_mapping.py b/openstackquery/mappings/hypervisor_mapping.py index 4ff4344..7e8774e 100644 --- a/openstackquery/mappings/hypervisor_mapping.py +++ b/openstackquery/mappings/hypervisor_mapping.py @@ -5,14 +5,10 @@ from openstackquery.enums.query_presets import ( QueryPresetsGeneric, QueryPresetsString, - QueryPresetsInteger, ) from openstackquery.handlers.client_side_handler_generic import ( ClientSideHandlerGeneric, ) -from openstackquery.handlers.client_side_handler_integer import ( - ClientSideHandlerInteger, -) from openstackquery.handlers.client_side_handler_string import ClientSideHandlerString from openstackquery.handlers.server_side_handler import ServerSideHandler from openstackquery.mappings.mapping_interface import MappingInterface @@ -72,19 +68,6 @@ def get_client_side_handlers() -> QueryClientSideHandlers: corresponding to valid preset-property pairs. These filter functions can be used to filter results after listing all hypervisors. """ - integer_props = [ - HypervisorProperties.HYPERVISOR_DISK_USED, - HypervisorProperties.HYPERVISOR_DISK_FREE, - HypervisorProperties.HYPERVISOR_DISK_SIZE, - HypervisorProperties.HYPERVISOR_MEMORY_SIZE, - HypervisorProperties.HYPERVISOR_MEMORY_USED, - HypervisorProperties.HYPERVISOR_MEMORY_FREE, - HypervisorProperties.HYPERVISOR_VCPUS, - HypervisorProperties.HYPERVISOR_VCPUS_USED, - # HypervisorProperties.HYPERVISOR_SERVER_COUNT, # Deprecated, use server query - # HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD, - ] - return QueryClientSideHandlers( # set generic query preset mappings generic_handler=ClientSideHandlerGeneric( @@ -108,12 +91,5 @@ def get_client_side_handlers() -> QueryClientSideHandlers: # set datetime query preset mappings datetime_handler=None, # set integer query preset mappings - integer_handler=ClientSideHandlerInteger( - { - QueryPresetsInteger.LESS_THAN: integer_props, - QueryPresetsInteger.GREATER_THAN: integer_props, - QueryPresetsInteger.LESS_THAN_OR_EQUAL_TO: integer_props, - QueryPresetsInteger.GREATER_THAN_OR_EQUAL_TO: integer_props, - } - ), + integer_handler=None, ) diff --git a/openstackquery/runners/hypervisor_runner.py b/openstackquery/runners/hypervisor_runner.py index b7d2295..0c76610 100644 --- a/openstackquery/runners/hypervisor_runner.py +++ b/openstackquery/runners/hypervisor_runner.py @@ -1,9 +1,7 @@ -import json import logging -from typing import Dict, List, Optional +from typing import List, Optional from openstack.compute.v2.hypervisor import Hypervisor -from osc_placement.http import SessionClient as PlacementClient from openstackquery.aliases import OpenstackResourceObj, ServerSideFilters from openstackquery.openstack_connection import OpenstackConnection @@ -29,49 +27,6 @@ def parse_meta_params(self, conn: OpenstackConnection, **kwargs): logger.debug("HypervisorQuery has no meta-params available") return super().parse_meta_params(conn, **kwargs) - def _populate_placement_info( - self, conn: OpenstackConnection, hypervisors: List - ) -> List: - """ - Adds resource usage stats to the hypervisors - :param conn: Openstack connecion - :param hypervisors: List of hypervisors - :return: List of hypervisors with additional resource usage stats - """ - client = PlacementClient( - api_version="1.6", - session=conn.session, - ks_filter={"service_type": "placement"}, - ) - - for hypervisor in hypervisors: - hypervisor.resources = self._get_usage_info(conn, client, hypervisor) - - return hypervisors - - def _get_usage_info( - self, conn: OpenstackConnection, client: PlacementClient, hypervisor: Hypervisor - ) -> Dict: - """ - Get usage stats from the openstack placement api - :param conn: Openstack connection - :param client: osc_placement session client - :param hypervisor: Openstack hypervisor - :return: resource usage for the hypervisor - """ - resources = conn.placement.resource_provider_inventories(hypervisor.id) - usages = client.request("get", f"/resource_providers/{hypervisor.id}/usages") - usages = json.loads(usages.text).get("usages") - usage_info = {} - for i in resources: - usage_info[i.resource_class] = { - "total": i.total, - "usage": usages.get(i.resource_class), - "free": i.total - usages.get(i.resource_class), - } - - return usage_info - # pylint: disable=unused-argument def run_query( self, @@ -95,8 +50,6 @@ def run_query( "running openstacksdk command conn.compute.hypervisors(%s)", ",".join(f"{key}={value}" for key, value in filter_kwargs.items()), ) - hypervisors = RunnerUtils.run_paginated_query( + return RunnerUtils.run_paginated_query( conn.compute.hypervisors, self._page_marker_prop_func, filter_kwargs ) - - return self._populate_placement_info(conn, hypervisors) diff --git a/tests/enums/props/test_hypervisor_properties.py b/tests/enums/props/test_hypervisor_properties.py index 7b16883..e5f91e5 100644 --- a/tests/enums/props/test_hypervisor_properties.py +++ b/tests/enums/props/test_hypervisor_properties.py @@ -37,66 +37,6 @@ def test_get_marker_prop_func(mock_get_prop_mapping): assert val == mock_get_prop_mapping.return_value -@pytest.mark.parametrize( - "val", - [ - "hypervisor_disk_free", - "Hypervisor_Disk_Free", - "HyPeRvIsOr_DiSk_FrEe", - "local_disk_free", - "free_disk_gb", - ], -) -def test_hypervisor_disk_free_serialization(val): - """ - Tests that variants of HYPERVISOR_DISK_FREE can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_DISK_FREE - ) - - -@pytest.mark.parametrize( - "val", - [ - "hypervisor_disk_size", - "Hypervisor_Disk_Size", - "HyPeRvIsOr_DiSk_SiZe", - "local_disk_size", - "local_gb", - ], -) -def test_hypervisor_disk_size_serialization(val): - """ - Tests that variants of HYPERVISOR_DISK_SIZE can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_DISK_SIZE - ) - - -@pytest.mark.parametrize( - "val", - [ - "hypervisor_disk_used", - "Hypervisor_Disk_Used", - "HyPeRvIsOr_DiSk_UsEd", - "local_disk_used", - "local_disk_used", - ], -) -def test_hypervisor_disk_used_serialization(val): - """ - Tests that variants of HYPERVISOR_DISK_USED can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_DISK_USED - ) - - @pytest.mark.parametrize( "val", ["hypervisor_id", "Hypervisor_ID", "HyPeRvIsOr_Id", "id", "uuid", "host_id"], @@ -119,66 +59,6 @@ def test_hypervisor_ip_serialization(val): assert HypervisorProperties.from_string(val) is HypervisorProperties.HYPERVISOR_IP -@pytest.mark.parametrize( - "val", - [ - "hypervisor_memory_free", - "Hypervisor_Memory_Free", - "HyPeRvIsOr_MeMoRy_FrEe", - "memory_free", - "free_ram_mb", - ], -) -def test_hypervisor_memory_free_serialization(val): - """ - Tests that variants of HYPERVISOR_MEMORY_FREE can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_MEMORY_FREE - ) - - -@pytest.mark.parametrize( - "val", - [ - "hypervisor_memory_size", - "Hypervisor_Memory_Size", - "HyPeRvIsOr_MeMoRy_SiZe", - "memory_size", - "memory_mb", - ], -) -def test_hypervisor_memory_size_serialization(val): - """ - Tests that variants of HYPERVISOR_MEMORY_SIZE can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_MEMORY_SIZE - ) - - -@pytest.mark.parametrize( - "val", - [ - "hypervisor_memory_used", - "Hypervisor_Memory_Used", - "HyPeRvIsOr_MeMoRy_UsEd", - "memory_used", - "memory_mb_used", - ], -) -def test_hypervisor_memory_used_serialization(val): - """ - Tests that variants of HYPERVISOR_MEMORY_USED can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_MEMORY_USED - ) - - @pytest.mark.parametrize( "val", ["hypervisor_name", "Hypervisor_Name", "HyPeRvIsOr_NaMe", "name", "host_name"], @@ -226,43 +106,6 @@ def test_hypervisor_status_serialization(val): ) -@pytest.mark.parametrize( - "val", - [ - "hypervisor_vcpus", - "Hypervisor_VCPUs", - "HyPeRvIsOr_VcPuS", - "vcpus", - ], -) -def test_hypervisor_vcpus_serialization(val): - """ - Tests that variants of HYPERVISOR_VCPUS can be serialized - """ - assert ( - HypervisorProperties.from_string(val) is HypervisorProperties.HYPERVISOR_VCPUS - ) - - -@pytest.mark.parametrize( - "val", - [ - "hypervisor_vcpus_used", - "Hypervisor_VCPUs_Used", - "HyPeRvIsOr_VcPuS_uSeD", - "vcpus_used", - ], -) -def test_hypervisor_vcpus_used_serialization(val): - """ - Tests that variants of HYPERVISOR_VCPUS_USED can be serialized - """ - assert ( - HypervisorProperties.from_string(val) - is HypervisorProperties.HYPERVISOR_VCPUS_USED - ) - - @pytest.mark.parametrize( "val", [ diff --git a/tests/mappings/test_hypervisor_mapping.py b/tests/mappings/test_hypervisor_mapping.py index 93d93fa..32edb24 100644 --- a/tests/mappings/test_hypervisor_mapping.py +++ b/tests/mappings/test_hypervisor_mapping.py @@ -3,7 +3,6 @@ from openstackquery.enums.query_presets import ( QueryPresetsGeneric, QueryPresetsString, - QueryPresetsInteger, ) from openstackquery.handlers.server_side_handler import ServerSideHandler from openstackquery.mappings.hypervisor_mapping import HypervisorMapping @@ -71,31 +70,6 @@ def test_client_side_handlers_datetime(): assert not handler -def test_client_side_handlers_integer(client_side_test_mappings): - """ - Tests client side handler mappings are correct, and line up to the expected - client side params for integer presets - """ - integer_prop_list = [ - HypervisorProperties.HYPERVISOR_DISK_USED, - HypervisorProperties.HYPERVISOR_DISK_FREE, - HypervisorProperties.HYPERVISOR_DISK_SIZE, - HypervisorProperties.HYPERVISOR_MEMORY_SIZE, - HypervisorProperties.HYPERVISOR_MEMORY_USED, - HypervisorProperties.HYPERVISOR_MEMORY_FREE, - HypervisorProperties.HYPERVISOR_VCPUS, - HypervisorProperties.HYPERVISOR_VCPUS_USED, - ] - handler = HypervisorMapping.get_client_side_handlers().integer_handler - mappings = { - QueryPresetsInteger.LESS_THAN: integer_prop_list, - QueryPresetsInteger.LESS_THAN_OR_EQUAL_TO: integer_prop_list, - QueryPresetsInteger.GREATER_THAN: integer_prop_list, - QueryPresetsInteger.GREATER_THAN_OR_EQUAL_TO: integer_prop_list, - } - client_side_test_mappings(handler, mappings) - - def test_get_chain_mappings(): """ Tests get_chain_mapping outputs correctly diff --git a/tests/runners/test_hypervisor_runner.py b/tests/runners/test_hypervisor_runner.py index 67ad650..b0ddb08 100644 --- a/tests/runners/test_hypervisor_runner.py +++ b/tests/runners/test_hypervisor_runner.py @@ -24,11 +24,9 @@ def test_parse_query_params(instance): ) -@patch("openstackquery.runners.hypervisor_runner.json.loads") @patch("openstackquery.runners.runner_utils.RunnerUtils.run_paginated_query") def test_run_query_no_server_filters( mock_run_paginated_query, - mock_json_loads, instance, mock_marker_prop_func, ): @@ -49,26 +47,6 @@ def test_run_query_no_server_filters( mock_connection = MagicMock() - vcpu_resource_class = MagicMock() - vcpu_resource_class.resource_class = "VCPU" - vcpu_resource_class.total = 128 - memory_resource_class = MagicMock() - memory_resource_class.resource_class = "MEMORY_MB" - memory_resource_class.total = 515264 - disk_resource_class = MagicMock() - disk_resource_class.resource_class = "DISK_GB" - disk_resource_class.total = 3510 - - mock_connection.placement.resource_provider_inventories.return_value = ( - vcpu_resource_class, - memory_resource_class, - disk_resource_class, - ) - - mock_json_loads.return_value = { - "usages": {"VCPU": 4, "MEMORY_MB": 8192, "DISK_GB": 10} - } - res = instance.run_query( mock_connection, filter_kwargs=None, @@ -80,17 +58,4 @@ def test_run_query_no_server_filters( {"details": True}, ) - assert mock_json_loads.call_count == 2 - assert res == mock_hv_list - - assert mock_hv1.resources == { - "VCPU": {"total": 128, "usage": 4, "free": 124}, - "MEMORY_MB": {"total": 515264, "usage": 8192, "free": 507072}, - "DISK_GB": {"total": 3510, "usage": 10, "free": 3500}, - } - assert mock_hv2.resources == { - "VCPU": {"total": 128, "usage": 4, "free": 124}, - "MEMORY_MB": {"total": 515264, "usage": 8192, "free": 507072}, - "DISK_GB": {"total": 3510, "usage": 10, "free": 3500}, - } From 237d9105356632d4af264512eb1eeda27fef42e3 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Fri, 3 Jan 2025 15:45:20 +0000 Subject: [PATCH 2/3] WIP: Add placement API to support getting HV/provider resources In Yoga+ placement should be used for determining usage...etc. with Horizon having switched to it upstream. Unfortunately, we don't have full support in the latest SDK for getting usage, so add some static methods and data classes to support this until we/someone else upstream the work. --- openstackquery/__init__.py | 1 + openstackquery/api/query_objects.py | 8 ++ .../enums/props/placement_properties.py | 79 +++++++++++ .../handlers/server_side_handler.py | 2 + openstackquery/mappings/placement_mapping.py | 121 +++++++++++++++++ openstackquery/runners/placement_runner.py | 127 ++++++++++++++++++ .../structs/resource_provider_usage.py | 23 ++++ 7 files changed, 361 insertions(+) create mode 100644 openstackquery/enums/props/placement_properties.py create mode 100644 openstackquery/mappings/placement_mapping.py create mode 100644 openstackquery/runners/placement_runner.py create mode 100644 openstackquery/structs/resource_provider_usage.py diff --git a/openstackquery/__init__.py b/openstackquery/__init__.py index 55e7261..73c0f5f 100644 --- a/openstackquery/__init__.py +++ b/openstackquery/__init__.py @@ -7,6 +7,7 @@ ProjectQuery, ImageQuery, HypervisorQuery, + PlacementQuery, ) # Create logger diff --git a/openstackquery/api/query_objects.py b/openstackquery/api/query_objects.py index d522b9c..0db9460 100644 --- a/openstackquery/api/query_objects.py +++ b/openstackquery/api/query_objects.py @@ -5,6 +5,7 @@ from openstackquery.mappings.hypervisor_mapping import HypervisorMapping from openstackquery.mappings.image_mapping import ImageMapping from openstackquery.mappings.mapping_interface import MappingInterface +from openstackquery.mappings.placement_mapping import PlacementMapping from openstackquery.mappings.project_mapping import ProjectMapping from openstackquery.mappings.server_mapping import ServerMapping from openstackquery.mappings.user_mapping import UserMapping @@ -70,3 +71,10 @@ def HypervisorQuery() -> "QueryAPI": Simple helper function to setup a query using a factory """ return get_common(HypervisorMapping) + + +def PlacementQuery() -> "QueryAPI": + """ + Simple helper function to setup a query using a factory + """ + return get_common(PlacementMapping) diff --git a/openstackquery/enums/props/placement_properties.py b/openstackquery/enums/props/placement_properties.py new file mode 100644 index 0000000..7761a79 --- /dev/null +++ b/openstackquery/enums/props/placement_properties.py @@ -0,0 +1,79 @@ +from enum import auto +from typing import Dict, Optional + +from openstackquery.enums.props.prop_enum import PropEnum, PropFunc +from openstackquery.exceptions.query_property_mapping_error import ( + QueryPropertyMappingError, +) + + +class PlacementProperties(PropEnum): + """ + An enum class for currently used placement properties + """ + + RESOURCE_PROVIDER_ID = auto() + RESOURCE_PROVIDER_NAME = auto() + VCPUS_USED = auto() + VCPUS_AVAIL = auto() + MEMORY_MB_USED = auto() + MEMORY_MB_AVAIL = auto() + DISK_GB_USED = auto() + DISK_GB_AVAIL = auto() + + @staticmethod + def _get_aliases() -> Dict: + """ + A method that returns all valid string alias mappings + """ + return { + PlacementProperties.RESOURCE_PROVIDER_ID: [ + "resource_provider_id", + "resource_provider_uuid", + "id", + ], + PlacementProperties.RESOURCE_PROVIDER_NAME: [ + "resource_name", + "name", + "provider_name", + ], + PlacementProperties.VCPUS_USED: ["vcpus_used"], + PlacementProperties.VCPUS_AVAIL: ["vcpus_avail"], + PlacementProperties.MEMORY_MB_USED: ["memory_mb_used"], + PlacementProperties.MEMORY_MB_AVAIL: ["memory_mb_avail"], + PlacementProperties.DISK_GB_USED: ["disk_gb_used"], + PlacementProperties.DISK_GB_AVAIL: ["disk_gb_avail"], + } + + @staticmethod + def get_prop_mapping(prop) -> Optional[PropFunc]: + """ + Method that returns the property function if function mapping exists for a given Hypervisor Enum + how to get specified property from a ResourceProviderUsage object + :param prop: A HypervisorProperty Enum for which a function may exist for + """ + mapping = { + PlacementProperties.RESOURCE_PROVIDER_ID: lambda a: a["id"], + PlacementProperties.RESOURCE_PROVIDER_NAME: lambda a: a["name"], + PlacementProperties.VCPUS_AVAIL: lambda a: a["vcpu_avail"], + PlacementProperties.MEMORY_MB_AVAIL: lambda a: a["memory_mb_avail"], + PlacementProperties.DISK_GB_AVAIL: lambda a: a["disk_gb_avail"], + PlacementProperties.VCPUS_USED: lambda a: a["vcpu_used"], + PlacementProperties.MEMORY_MB_USED: lambda a: a["memory_mb_used"], + PlacementProperties.DISK_GB_USED: lambda a: a["disk_gb_used"], + } + try: + return mapping[prop] + except KeyError as exp: + raise QueryPropertyMappingError( + f"Error: failed to get property mapping, property {prop.name} is not supported in PlacementProperties" + ) from exp + + @staticmethod + def get_marker_prop_func(): + """ + A getter method to return marker property function for pagination + """ + return PlacementProperties.get_prop_mapping( + PlacementProperties.RESOURCE_PROVIDER_ID + ) diff --git a/openstackquery/handlers/server_side_handler.py b/openstackquery/handlers/server_side_handler.py index 50e3fc6..391744f 100644 --- a/openstackquery/handlers/server_side_handler.py +++ b/openstackquery/handlers/server_side_handler.py @@ -87,6 +87,8 @@ def get_filters( try: filters = filter_func(**params) except (KeyError, TypeError) as err: + # Dev note: your lambda must take "value" as the lambda + # argument if you arrive here adding new mappings raise QueryPresetMappingError( "Preset Argument Error: failed to build server-side openstacksdk filters for preset:prop: " f"'{preset.name}':'{prop.name}' " diff --git a/openstackquery/mappings/placement_mapping.py b/openstackquery/mappings/placement_mapping.py new file mode 100644 index 0000000..c7c9951 --- /dev/null +++ b/openstackquery/mappings/placement_mapping.py @@ -0,0 +1,121 @@ +from typing import Type + +from openstackquery.enums.props.hypervisor_properties import HypervisorProperties +from openstackquery.enums.props.placement_properties import PlacementProperties +from openstackquery.enums.props.prop_enum import PropEnum +from openstackquery.enums.query_presets import ( + QueryPresetsGeneric, + QueryPresetsString, + QueryPresetsInteger, +) +from openstackquery.handlers.client_side_handler_generic import ( + ClientSideHandlerGeneric, +) +from openstackquery.handlers.client_side_handler_integer import ( + ClientSideHandlerInteger, +) +from openstackquery.handlers.client_side_handler_string import ClientSideHandlerString +from openstackquery.handlers.server_side_handler import ServerSideHandler +from openstackquery.mappings.mapping_interface import MappingInterface +from openstackquery.runners.placement_runner import PlacementRunner + +from openstackquery.runners.runner_wrapper import RunnerWrapper +from openstackquery.structs.query_client_side_handlers import QueryClientSideHandlers + + +class PlacementMapping(MappingInterface): + """ + Mapping class for querying Openstack placement and resource objects + Define property mappings, kwarg mappings and filter function mappings, + and runner mapping related to placement and resources here + """ + + @staticmethod + def get_chain_mappings(): + """ + Should return a dictionary containing property pairs mapped to query mappings. + This is used to define how to chain results from this query to other possible queries + """ + return { + PlacementProperties.RESOURCE_PROVIDER_NAME: HypervisorProperties.HYPERVISOR_NAME + } + + @staticmethod + def get_runner_mapping() -> Type[RunnerWrapper]: + """ + Returns a mapping to associated Runner class for the Query (placement and resourceRunner) + """ + return PlacementRunner + + @staticmethod + def get_prop_mapping() -> Type[PropEnum]: + """ + Returns a mapping of valid presets for server side attributes (placement and resourceProperties) + """ + return PlacementProperties + + @staticmethod + def get_server_side_handler() -> ServerSideHandler: + """ + method to configure a server handler which can be used to get 'filter' keyword arguments that + can be passed to openstack function conn.placement.resource_providers() to filter results + Valid filters documented here: + https://docs.openstack.org/openstacksdk/latest/user/proxies/placement.html + """ + return ServerSideHandler( + { + QueryPresetsGeneric.EQUAL_TO: { + PlacementProperties.RESOURCE_PROVIDER_ID: lambda value: { + "id": value + }, + PlacementProperties.RESOURCE_PROVIDER_NAME: lambda value: { + "name": value + }, + } + } + ) + + @staticmethod + def get_client_side_handlers() -> QueryClientSideHandlers: + """ + method to configure a set of client-side handlers which can be used to get local filter functions + corresponding to valid preset-property pairs. These filter functions can be used to filter results after + listing all placement and resources. + """ + integer_prop_list = [ + PlacementProperties.VCPUS_AVAIL, + PlacementProperties.MEMORY_MB_AVAIL, + PlacementProperties.DISK_GB_AVAIL, + PlacementProperties.VCPUS_USED, + PlacementProperties.MEMORY_MB_USED, + PlacementProperties.DISK_GB_USED, + ] + + return QueryClientSideHandlers( + generic_handler=ClientSideHandlerGeneric( + { + QueryPresetsGeneric.EQUAL_TO: ["*"], + QueryPresetsGeneric.NOT_EQUAL_TO: ["*"], + QueryPresetsGeneric.ANY_IN: ["*"], + QueryPresetsGeneric.NOT_ANY_IN: ["*"], + } + ), + # set string query preset mappings + string_handler=ClientSideHandlerString( + { + QueryPresetsString.MATCHES_REGEX: [ + PlacementProperties.RESOURCE_PROVIDER_ID, + PlacementProperties.RESOURCE_PROVIDER_NAME, + ] + } + ), + datetime_handler=None, + integer_handler=ClientSideHandlerInteger( + { + QueryPresetsInteger.LESS_THAN: integer_prop_list, + QueryPresetsInteger.LESS_THAN_OR_EQUAL_TO: integer_prop_list, + QueryPresetsInteger.GREATER_THAN: integer_prop_list, + QueryPresetsInteger.GREATER_THAN_OR_EQUAL_TO: integer_prop_list, + } + ), + ) diff --git a/openstackquery/runners/placement_runner.py b/openstackquery/runners/placement_runner.py new file mode 100644 index 0000000..e64566d --- /dev/null +++ b/openstackquery/runners/placement_runner.py @@ -0,0 +1,127 @@ +import logging +from typing import List, Optional, Dict + +from openstack import exceptions, utils +from openstack.placement.v1.resource_provider import ResourceProvider + +from openstackquery.aliases import ( + OpenstackResourceObj, + ServerSideFilter, +) +from openstackquery.openstack_connection import OpenstackConnection +from openstackquery.runners.runner_wrapper import RunnerWrapper +from openstackquery.structs.resource_provider_usage import ResourceProviderUsage + +logger = logging.getLogger(__name__) + + +class PlacementRunner(RunnerWrapper): + """ + Runner class for openstack Hypervisor resource + HypervisorRunner encapsulates running any openstacksdk Hypervisor commands + """ + + RESOURCE_TYPE = ResourceProvider + + def parse_meta_params(self, conn: OpenstackConnection, **kwargs) -> Dict[str, str]: + """ + Placement runner has no meta-params available + """ + logger.debug("PlacementQuery has no meta-params available") + return super().parse_meta_params(conn, **kwargs) + + def _convert_to_custom_obj( + self, conn: OpenstackConnection, obj: ResourceProvider + ) -> OpenstackResourceObj: + """ + Converts an openstacksdk ResourceProvider object to a ResourceProviderUsage object + including populating the available and used resources from the placement API + :param conn: Openstack connection + :param obj: Openstack placement resource provider object + :return: A ResourceProviderUsage object + """ + usage = self._get_usage_info(conn, obj) + avail = self._get_availability_info(conn, obj) + return ResourceProviderUsage( + id=obj.id, + name=obj.name, + vcpu_used=usage["VCPU"], + memory_mb_used=usage["MEMORY_MB"], + disk_gb_used=usage["DISK_GB"], + vcpu_avail=avail["VCPU"], + memory_mb_avail=avail["MEMORY_MB"], + disk_gb_avail=avail["DISK_GB"], + ) + + @staticmethod + def _get_availability_info( + conn: OpenstackConnection, resource_provider_obj: ResourceProvider + ) -> Dict: + """ + Gets availability stats for a given placement resource provider + across the following resource classes: VCPU, MEMORY_MB, DISK_GB + :param conn: Openstack connection + :param resource_provider_obj: Openstack placement resource provider object + :return: A dictionary with the summed availability stats using the class name as a key + """ + summed_classes = {} + for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB"]: + placement_inventories = conn.placement.resource_provider_inventories( + resource_provider_obj, resource_class=resource_class + ) + # A resource provider can have n number of inventories for a given resource class + if not placement_inventories: + logger.warning( + "No available resources found for resource provider: %s", + resource_provider_obj.id, + ) + summed_classes[resource_class] = 0 + else: + summed_classes[resource_class] = sum( + i["total"] for i in placement_inventories + ) + return summed_classes + + @staticmethod + def _get_usage_info( + conn: OpenstackConnection, resource_provider_obj: ResourceProvider + ) -> Dict: + """ + Gets usage stats for a given placement resource provider + :param conn: Openstack connection + :param resource_provider_obj: Openstack placement resource provider object + :return: A ResourceProviderUsage object with usage stats + """ + # The following should be up-streamed to openstacksdk at some point + # It is based on the existing `resource_provider.py:fetch_aggregates` method + # found in the OpenStack SDK + url = utils.urljoin( + ResourceProvider.base_path, resource_provider_obj.id, "usages" + ) + + response = conn.session.get(url, endpoint_filter={"service_type": "placement"}) + exceptions.raise_from_response(response) + return response.json()["usages"] + + # pylint: disable=unused-argument + def run_query( + self, + conn: OpenstackConnection, + filter_kwargs: Optional[ServerSideFilter] = None, + **kwargs, + ) -> List[OpenstackResourceObj]: + """ + This method runs the query by running openstacksdk placement commands + + :param conn: An OpenstackConnection object - used to connect to openstacksdk + :param filter_kwargs: An Optional list of filter kwargs to pass to the openstacksdk command + """ + logger.debug( + "running openstacksdk command conn.placement.resource_providers(%s)", + ",".join(f"{key}={value}" for key, value in filter_kwargs.items()), + ) + resource_providers = conn.placement.resource_providers(**filter_kwargs) + return [ + self._convert_to_custom_obj(conn, provider) + for provider in resource_providers + ] diff --git a/openstackquery/structs/resource_provider_usage.py b/openstackquery/structs/resource_provider_usage.py new file mode 100644 index 0000000..4146e07 --- /dev/null +++ b/openstackquery/structs/resource_provider_usage.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + + +# pylint: disable=too-many-instance-attributes +@dataclass +class ResourceProviderUsage: + """ + Upstream has a resource provider class which only provides available resources. + Current usage is not supported at all. Instead, create a custom class to store + usage and total information until upstream updates its resource provider class. + """ + + # Lower case to maintain compatibility with existing ResourceProvider object + name: str + id: str + + vcpu_avail: int + memory_mb_avail: int + disk_gb_avail: int + + vcpu_used: int + memory_mb_used: int + disk_gb_used: int From 7e80e573af60a5167394c7efe5f6549477efaeca Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Fri, 3 Jan 2025 15:55:10 +0000 Subject: [PATCH 3/3] Bump to v1.0.0 and update author Update author to reflect entire group and bump to v1 with latest breaking changes removing various HV presets --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 61e079e..1d523f4 100644 --- a/setup.py +++ b/setup.py @@ -7,9 +7,9 @@ setup( name="openstackquery", - version="0.1.4", - author="Anish Mudaraddi", - author_email="", + version="1.0.0", + author="STFC Cloud Team", + author_email="", description=DESCRIPTION, long_description=LONG_DESCRIPTION, packages=find_packages(),