WIP: Add placement API to support getting HV/provider resources

In Yoga+ placement should be used for determining usage...etc. with
Horizon having switched to it upstream. Unfortunately, we don't have
full support in the latest SDK for getting usage, so add some static
methods and data classes to support this until we/someone else upstream
the work.

openstackquery/__init__.py

@@ -7,6 +7,7 @@
     ProjectQuery,
     ImageQuery,
     HypervisorQuery,
+    PlacementQuery,
 )
 
 # Create logger

openstackquery/api/query_objects.py

@@ -5,6 +5,7 @@
 from openstackquery.mappings.hypervisor_mapping import HypervisorMapping
 from openstackquery.mappings.image_mapping import ImageMapping
 from openstackquery.mappings.mapping_interface import MappingInterface
+from openstackquery.mappings.placement_mapping import PlacementMapping
 from openstackquery.mappings.project_mapping import ProjectMapping
 from openstackquery.mappings.server_mapping import ServerMapping
 from openstackquery.mappings.user_mapping import UserMapping
@@ -70,3 +71,10 @@ def HypervisorQuery() -> "QueryAPI":
     Simple helper function to setup a query using a factory
     """
     return get_common(HypervisorMapping)
+
+
+def PlacementQuery() -> "QueryAPI":
+    """
+    Simple helper function to setup a query using a factory
+    """
+    return get_common(PlacementMapping)

openstackquery/enums/props/placement_properties.py

@@ -0,0 +1,79 @@
+from enum import auto
+from typing import Dict, Optional
+
+from openstackquery.enums.props.prop_enum import PropEnum, PropFunc
+from openstackquery.exceptions.query_property_mapping_error import (
+    QueryPropertyMappingError,
+)
+
+
+class PlacementProperties(PropEnum):
+    """
+    An enum class for currently used placement properties
+    """
+
+    RESOURCE_PROVIDER_ID = auto()
+    RESOURCE_PROVIDER_NAME = auto()
+    VCPUS_USED = auto()
+    VCPUS_AVAIL = auto()
+    MEMORY_MB_USED = auto()
+    MEMORY_MB_AVAIL = auto()
+    DISK_GB_USED = auto()
+    DISK_GB_AVAIL = auto()
+
+    @staticmethod
+    def _get_aliases() -> Dict:
+        """
+        A method that returns all valid string alias mappings
+        """
+        return {
+            PlacementProperties.RESOURCE_PROVIDER_ID: [
+                "resource_provider_id",
+                "resource_provider_uuid",
+                "id",
+            ],
+            PlacementProperties.RESOURCE_PROVIDER_NAME: [
+                "resource_name",
+                "name",
+                "provider_name",
+            ],
+            PlacementProperties.VCPUS_USED: ["vcpus_used"],
+            PlacementProperties.VCPUS_AVAIL: ["vcpus_avail"],
+            PlacementProperties.MEMORY_MB_USED: ["memory_mb_used"],
+            PlacementProperties.MEMORY_MB_AVAIL: ["memory_mb_avail"],
+            PlacementProperties.DISK_GB_USED: ["disk_gb_used"],
+            PlacementProperties.DISK_GB_AVAIL: ["disk_gb_avail"],
+        }
+
+    @staticmethod
+    def get_prop_mapping(prop) -> Optional[PropFunc]:
+        """
+        Method that returns the property function if function mapping exists for a given Hypervisor Enum
+        how to get specified property from a ResourceProviderUsage object
+        :param prop: A HypervisorProperty Enum for which a function may exist for
+        """
+        mapping = {
+            PlacementProperties.RESOURCE_PROVIDER_ID: lambda a: a["id"],
+            PlacementProperties.RESOURCE_PROVIDER_NAME: lambda a: a["name"],
+            PlacementProperties.VCPUS_AVAIL: lambda a: a["VCPU_AVAIL"],
+            PlacementProperties.MEMORY_MB_AVAIL: lambda a: a["MEMORY_MB_AVAIL"],
+            PlacementProperties.DISK_GB_AVAIL: lambda a: a["DISK_GB_AVAIL"],
+            PlacementProperties.VCPUS_USED: lambda a: a["VCPU_USED"],
+            PlacementProperties.MEMORY_MB_USED: lambda a: a["MEMORY_MB_USED"],
+            PlacementProperties.DISK_GB_USED: lambda a: a["DISK_GB_USED"],
+        }
+        try:
+            return mapping[prop]
+        except KeyError as exp:
+            raise QueryPropertyMappingError(
+                f"Error: failed to get property mapping, property {prop.name} is not supported in PlacementProperties"
+            ) from exp
+
+    @staticmethod
+    def get_marker_prop_func():
+        """
+        A getter method to return marker property function for pagination
+        """
+        return PlacementProperties.get_prop_mapping(
+            PlacementProperties.RESOURCE_PROVIDER_ID
+        )

openstackquery/handlers/server_side_handler.py

@@ -87,6 +87,8 @@ def get_filters(
         try:
             filters = filter_func(**params)
         except (KeyError, TypeError) as err:
+            # Dev note: your lambda must take "value" as the lambda
+            # argument if you arrive here adding new mappings
             raise QueryPresetMappingError(
                 "Preset Argument Error: failed to build server-side openstacksdk filters for preset:prop: "
                 f"'{preset.name}':'{prop.name}' "

openstackquery/mappings/placement_mapping.py

@@ -0,0 +1,123 @@
+from typing import Type
+
+from openstackquery.enums.props.hypervisor_properties import HypervisorProperties
+from openstackquery.enums.props.placement_properties import PlacementProperties
+from openstackquery.enums.props.prop_enum import PropEnum
+from openstackquery.enums.query_presets import (
+    QueryPresetsGeneric,
+    QueryPresetsString,
+    QueryPresetsInteger,
+)
+from openstackquery.handlers.client_side_handler_generic import (
+    ClientSideHandlerGeneric,
+)
+from openstackquery.handlers.client_side_handler_integer import (
+    ClientSideHandlerInteger,
+)
+from openstackquery.handlers.client_side_handler_string import ClientSideHandlerString
+from openstackquery.handlers.server_side_handler import ServerSideHandler
+from openstackquery.mappings.mapping_interface import MappingInterface
+from openstackquery.runners.placement_runner import PlacementRunner
+
+from openstackquery.runners.runner_wrapper import RunnerWrapper
+from openstackquery.structs.query_client_side_handlers import QueryClientSideHandlers
+
+
+class PlacementMapping(MappingInterface):
+    """
+    Mapping class for querying Openstack placement and resource objects
+    Define property mappings, kwarg mappings and filter function mappings,
+    and runner mapping related to placement and resources here
+    """
+
+    @staticmethod
+    def get_chain_mappings():
+        """
+        Should return a dictionary containing property pairs mapped to query mappings.
+        This is used to define how to chain results from this query to other possible queries
+        """
+        return {
+            PlacementProperties.RESOURCE_PROVIDER_NAME: HypervisorProperties.HYPERVISOR_NAME
+        }
+
+    @staticmethod
+    def get_runner_mapping() -> Type[RunnerWrapper]:
+        """
+        Returns a mapping to associated Runner class for the Query (placement and resourceRunner)
+        """
+        return PlacementRunner
+
+    @staticmethod
+    def get_prop_mapping() -> Type[PropEnum]:
+        """
+        Returns a mapping of valid presets for server side attributes (placement and resourceProperties)
+        """
+        return PlacementProperties
+
+    @staticmethod
+    def get_server_side_handler() -> ServerSideHandler:
+        """
+        method to configure a server handler which can be used to get 'filter' keyword arguments that
+        can be passed to openstack function conn.placement.resource_providers() to filter results for a valid preset-property
+        pair
+
+        valid filters documented here:
+            https://docs.openstack.org/openstacksdk/latest/user/proxies/placement.html
+        """
+        return ServerSideHandler(
+            {
+                QueryPresetsGeneric.EQUAL_TO: {
+                    PlacementProperties.RESOURCE_PROVIDER_ID: lambda value: {
+                        "id": value
+                    },
+                    PlacementProperties.RESOURCE_PROVIDER_NAME: lambda value: {
+                        "name": value
+                    },
+                }
+            }
+        )
+
+    @staticmethod
+    def get_client_side_handlers() -> QueryClientSideHandlers:
+        """
+        method to configure a set of client-side handlers which can be used to get local filter functions
+        corresponding to valid preset-property pairs. These filter functions can be used to filter results after
+        listing all placement and resources.
+        """
+        integer_prop_list = [
+            PlacementProperties.VCPUS_AVAIL,
+            PlacementProperties.MEMORY_MB_AVAIL,
+            PlacementProperties.DISK_GB_AVAIL,
+            PlacementProperties.VCPUS_USED,
+            PlacementProperties.MEMORY_MB_USED,
+            PlacementProperties.DISK_GB_USED,
+        ]
+
+        return QueryClientSideHandlers(
+            generic_handler=ClientSideHandlerGeneric(
+                {
+                    QueryPresetsGeneric.EQUAL_TO: ["*"],
+                    QueryPresetsGeneric.NOT_EQUAL_TO: ["*"],
+                    QueryPresetsGeneric.ANY_IN: ["*"],
+                    QueryPresetsGeneric.NOT_ANY_IN: ["*"],
+                }
+            ),
+            # set string query preset mappings
+            string_handler=ClientSideHandlerString(
+                {
+                    QueryPresetsString.MATCHES_REGEX: [
+                        PlacementProperties.RESOURCE_PROVIDER_ID,
+                        PlacementProperties.RESOURCE_PROVIDER_NAME,
+                    ]
+                }
+            ),
+            datetime_handler=None,
+            integer_handler=ClientSideHandlerInteger(
+                {
+                    QueryPresetsInteger.LESS_THAN: integer_prop_list,
+                    QueryPresetsInteger.LESS_THAN_OR_EQUAL_TO: integer_prop_list,
+                    QueryPresetsInteger.GREATER_THAN: integer_prop_list,
+                    QueryPresetsInteger.GREATER_THAN_OR_EQUAL_TO: integer_prop_list,
+                }
+            ),
+        )

openstackquery/runners/placement_runner.py

@@ -0,0 +1,132 @@
+import logging
+from typing import List, Optional, Dict
+
+from openstack.placement.v1.resource_provider import ResourceProvider
+
+from openstackquery.aliases import (
+    OpenstackResourceObj,
+    ServerSideFilters,
+    ServerSideFilter,
+)
+from openstackquery.openstack_connection import OpenstackConnection
+from openstackquery.runners.runner_utils import RunnerUtils
+from openstackquery.runners.runner_wrapper import RunnerWrapper
+from openstackquery.structs.resource_provider_usage import ResourceProviderUsage
+
+logger = logging.getLogger(__name__)
+
+
+class PlacementRunner(RunnerWrapper):
+    """
+    Runner class for openstack Hypervisor resource
+    HypervisorRunner encapsulates running any openstacksdk Hypervisor commands
+    """
+
+    RESOURCE_TYPE = ResourceProvider
+
+    def parse_meta_params(self, conn: OpenstackConnection, **kwargs):
+        """
+        This class has no meta-params available, so this method is a no-op
+        """
+        return super().parse_meta_params(conn, **kwargs)
+
+    def _convert_to_custom_obj(
+        self, conn: OpenstackConnection, obj: ResourceProvider
+    ) -> OpenstackResourceObj:
+        """
+        Converts an openstacksdk ResourceProvider object to a ResourceProviderUsage object
+        including populating the available and used resources from the placement API
+        :param conn: Openstack connection
+        :param obj: Openstack placement resource provider object
+        :return: A ResourceProviderUsage object
+        """
+        usage = self._get_usage_info(conn, obj)
+        avail = self._get_availability_info(conn, obj)
+        return ResourceProviderUsage(
+            id=obj.id,
+            name=obj.name,
+            VCPU_USED=usage["VCPU"],
+            MEMORY_MB_USED=usage["MEMORY_MB"],
+            DISK_GB_USED=usage["DISK_GB"],
+            VCPU_AVAIL=avail["VCPU"],
+            MEMORY_MB_AVAIL=avail["MEMORY_MB"],
+            DISK_GB_AVAIL=avail["DISK_GB"],
+        )
+
+    @staticmethod
+    def _get_availability_info(
+        conn: OpenstackConnection, resource_provider_obj: ResourceProvider
+    ) -> Dict:
+        """
+        Gets availability stats for a given placement resource provider
+        across the following resource classes: VCPU, MEMORY_MB, DISK_GB
+        :param conn: Openstack connection
+        :param resource_provider_obj: Openstack placement resource provider object
+        :return: A dictionary with the summed availability stats using the class name as a key
+        """
+        summed_classes = {}
+        for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB"]:
+            placement_inventories = conn.placement.resource_provider_inventories(
+                resource_provider_obj, resource_class=resource_class
+            )
+            # A resource provider can have n number of inventories for a given resource class
+            if not placement_inventories:
+                logger.warning(
+                    "No available resources found for resource provider: %s",
+                    resource_provider_obj.id,
+                )
+                summed_classes[resource_class] = 0
+            else:
+                summed_classes[resource_class] = sum(
+                    i["total"] for i in placement_inventories
+                )
+        return summed_classes
+
+    @staticmethod
+    def _get_usage_info(
+        conn: OpenstackConnection, resource_provider_obj: ResourceProvider
+    ) -> Dict:
+        """
+        Gets usage stats for a given placement resource provider
+        :param conn: Openstack connection
+        :param resource_provider_obj: Openstack placement resource provider object
+        :return: A ResourceProviderUsage object with usage stats
+        """
+        # The following should be up-streamed to openstacksdk at some point
+        # It is based on the existing `resource_provider.py:fetch_aggregates` method
+        # found in the OpenStack SDK
+        from openstack import exceptions, utils
+
+        url = utils.urljoin(
+            ResourceProvider.base_path, resource_provider_obj.id, "usages"
+        )
+
+        response = conn.session.get(url, endpoint_filter={"service_type": "placement"})
+        exceptions.raise_from_response(response)
+        return response.json()["usages"]
+
+    # pylint: disable=unused-argument
+    def run_query(
+        self,
+        conn: OpenstackConnection,
+        filter_kwargs: Optional[ServerSideFilter] = None,
+        **kwargs,
+    ) -> List[OpenstackResourceObj]:
+        """
+        This method runs the query by running openstacksdk commands
+
+        For HypervisorQuery, this command finds all hypervisors that match a given set of filter_kwargs
+        :param conn: An OpenstackConnection object - used to connect to openstacksdk
+        :param filter_kwargs: An Optional list of filter kwargs to pass to conn.compute.hypervisors()
+            to limit the hypervisors being returned.
+            - see https://docs.openstack.org/api-ref/compute/?expanded=list-hypervisors-detail
+        """
+        logger.debug(
+            "running openstacksdk command conn.placement.resource_providers(%s)",
+            ",".join(f"{key}={value}" for key, value in filter_kwargs.items()),
+        )
+        resource_providers = conn.placement.resource_providers(**filter_kwargs)
+        return [
+            self._convert_to_custom_obj(conn, provider)
+            for provider in resource_providers
+        ]

openstackquery/structs/resource_provider_usage.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+
+from openstack.placement.v1.resource_provider import ResourceProvider
+
+
+@dataclass
+class ResourceProviderUsage:
+    """
+    Upstream has a resource provider class which only provides available information
+    usage is not supported at all. Instead, create a custom class to store usage information
+    until upstream has a dedicated class for usage
+    """
+
+    # Lower case to maintain compatibility with existing ResourceProvider object
+    name: str
+    id: str
+
+    VCPU_AVAIL: int
+    MEMORY_MB_AVAIL: int
+    DISK_GB_AVAIL: int
+
+    VCPU_USED: int
+    MEMORY_MB_USED: int
+    DISK_GB_USED: int