From a5f049d65ed163502f65a2b191b6108dfe4be5b2 Mon Sep 17 00:00:00 2001 From: ekatchko Date: Thu, 16 Jul 2020 22:39:26 +0200 Subject: [PATCH 1/3] refactor(dev): simulate openstack and api with toml --- .gitignore | 2 + dummy_cloud.py | 245 +++++++++++++++++++++++++++ project_usage_exporter.py | 288 +++++++++----------------------- requirements.txt | 38 +++++ resources/credits_machines.toml | 29 ---- resources/dummy_cc.toml | 86 ++++++++++ resources/dummy_machines.toml | 32 ---- resources/dummy_weights.toml | 19 +++ resources/high_usage.toml | 16 -- resources/low_usage.toml | 16 -- test_os_outputs.py | 110 ++++++++++++ 11 files changed, 580 insertions(+), 301 deletions(-) create mode 100644 dummy_cloud.py create mode 100644 requirements.txt delete mode 100644 resources/credits_machines.toml create mode 100644 resources/dummy_cc.toml delete mode 100644 resources/dummy_machines.toml create mode 100644 resources/dummy_weights.toml delete mode 100644 resources/high_usage.toml delete mode 100644 resources/low_usage.toml create mode 100755 test_os_outputs.py diff --git a/.gitignore b/.gitignore index 1cefad5..84c6866 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.idea + *openrc* # Swap [._]*.s[a-v][a-z] diff --git a/dummy_cloud.py b/dummy_cloud.py new file mode 100644 index 0000000..63eee4d --- /dev/null +++ b/dummy_cloud.py @@ -0,0 +1,245 @@ +from datetime import datetime, timedelta + +import toml +from munch import Munch +import requests +from typing import ( + Tuple, + Union, + cast, +) +from enum import Enum +import json + +hour_timedelta = timedelta(hours=1) + + +class DummyCloud: + + def __init__(self, dummy_file, start=None): + self.dummy_file = dummy_file + self.dummy_values = toml.loads(self.dummy_file.read()) + if start is not None: + script_start = start + else: + script_start = datetime.now() + self.compute = Compute(self.dummy_values, script_start) + + def load_toml(self): + self.dummy_file.seek(0) + self.dummy_values = toml.loads(self.dummy_file.read()) + self.compute.reload(self.dummy_values) + + def list_projects(self, domain_id=None): + self.load_toml() + projects_return = [] + for domain_name, domain_content in self.dummy_values.items(): + if domain_id is not None and domain_content.get("domain_id", "UNKNOWN_DOMAIN_ID") != domain_id: + continue + projects_in_domain = domain_content.get("projects", []) + for project_in_domain in projects_in_domain: + project = Munch() + project.id = project_in_domain.get("project_id", "UNKNOWN_ID") + project.name = project_in_domain.get("project_name", "UNKNOWN_NAME") + projects_return.append(project) + return projects_return + + def get_domain(self, name_or_id): + self.load_toml() + for domain_name, domain_content in self.dummy_values.items(): + file_domain_id = domain_content.get("domain_id", "UNKNOWN_DOMAIN_ID") + if domain_name == name_or_id or file_domain_id == name_or_id: + return Munch(id=file_domain_id, name=domain_name) + return None + + +class ExistenceInformation(Enum): + NO_EXISTENCE = 0 + SINCE_SCRIPT_START = 1 + SINCE_DATETIME = 2 + BETWEEN_DATETIMES = 3 + + +class Compute: + + class DummyMachine: + """ + Representing a dummy machine causing usage to monitor. + :param name: Currently not used outside but might be in future, therefore leave it + :param cpus: Number of cpus the dummy machine is using. + :param ram: Amount of RAM [GiB] the machine is using. + :param existence: Determines whether the machine is *up* and its usage so far. In case + of True the machine is considered booted up the instant this script is started. In + case of False it hasn't been booted ever (no actual use case). + In case of a single datetime the machine is considered *up* since that moment (for + simplicity the timezone information are ignored). In case of a list of two datetimes + the machine is considered *up* the time in between. The first one must be + older/smaller than the second one and both but relative to the moment the script + started both may lie in the future or past. + """ + + def __init__(self, + cpus: int = 4, + ram: int = 8, + existence: Union[bool, datetime, Tuple[datetime, datetime]] = True, + metadata=None, + instance_id="UNKNOWN_ID"): + self.cpus = cpus + self.ram = ram + self.existence = existence + self.metadata = metadata + self.instance_id = instance_id + self.init_existence_information() + + def init_existence_information(self) -> None: + if self.cpus <= 0 or self.ram <= 0: + raise ValueError("`cpu` and `ram` must be positive") + if isinstance(self.existence, (list, tuple)): + if self.existence[0] > self.existence[1]: # type: ignore + raise ValueError( + "First existence-tuple datetime must be older than second one" + ) + # remove any timezone information + self.existence_information = ExistenceInformation.BETWEEN_DATETIMES + elif isinstance(self.existence, datetime): + self.existence_information = ExistenceInformation.SINCE_DATETIME + elif isinstance(self.existence, bool): + self.existence_information = ( + ExistenceInformation.SINCE_SCRIPT_START + if self.existence + else ExistenceInformation.NO_EXISTENCE + ) + else: + raise ValueError( + f"Invalid type for param `existence` (got {type(self.existence)}" + ) + + @property + def ram_mb(self) -> int: + return self.ram * 1024 + + def compute_server_info(self, requested_start_date, script_start) -> Munch: + requested_start_date = datetime.strptime(requested_start_date, "%Y-%m-%dT%H:%M:%S.%f") + now = datetime.now() + return_dict = Munch() + return_dict.hours = 0.0 + return_dict.vcpus = self.cpus + return_dict.memory_mb = self.ram_mb + return_dict.started_at = script_start.strftime("%Y-%m-%dT%H:%M:%S.%f") + return_dict.instance_id = self.instance_id + if self.existence_information is ExistenceInformation.SINCE_SCRIPT_START: + if requested_start_date > script_start: + hours_existence = (datetime.now() - requested_start_date) / hour_timedelta + else: + hours_existence = (datetime.now() - script_start) / hour_timedelta + return_dict.hours = hours_existence + elif self.existence_information is ExistenceInformation.NO_EXISTENCE: + return return_dict + elif self.existence_information is ExistenceInformation.SINCE_DATETIME: + # to satisfy `mypy` type checker + boot_datetime = cast(datetime, self.existence) + if requested_start_date > boot_datetime: + hours_existence = (datetime.now() - requested_start_date) / hour_timedelta + else: + hours_existence = (now - boot_datetime.replace(tzinfo=None)) / hour_timedelta + + # do not report negative usage in case the machine is not *booted yet* + return_dict.started_at = boot_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f") + if hours_existence > 0: + return_dict.hours = hours_existence + else: + # to satisfy `mypy` type checker + runtime_tuple = cast(Tuple[datetime, datetime], self.existence) + boot_datetime = cast(datetime, runtime_tuple[0].replace(tzinfo=None)) + shutdown_datetime = cast(datetime, runtime_tuple[1].replace(tzinfo=None)) + return_dict.started_at = boot_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f") + if boot_datetime > now: + # machine did not boot yet + hours_existence = 0.0 + elif shutdown_datetime < now: + # machine did run already and is considered down + if requested_start_date > boot_datetime: + hours_existence = (shutdown_datetime - requested_start_date) / hour_timedelta + else: + hours_existence = (shutdown_datetime - boot_datetime) / hour_timedelta + else: + # machine booted in the past but is still existing + if requested_start_date > boot_datetime: + hours_existence = (now - requested_start_date) / hour_timedelta + else: + hours_existence = (now - boot_datetime) / hour_timedelta + return_dict.hours = hours_existence + return return_dict + + def get_details(self): + return {"id": self.instance_id, "metadata": self.metadata} + + def __init__(self, dummy_values, script_start): + self.dummy_values = dummy_values + self.os_simple_tenant_usage_string = "/os-simple-tenant-usage/" + self.server_detail_all_tenants_string = "/servers/detail?all_tenants=false&project_id=" + self.script_start = script_start + + def reload(self, dummy_values): + self.dummy_values = dummy_values + + def get_tenant_usage(self, project, requested_start_date): + server_usages = [] + start = requested_start_date + stop = datetime.now() + + for toml_machine in project.get("machines", []): + machine = self.DummyMachine(toml_machine.get("cpus", 4), toml_machine.get("ram", 8), + toml_machine.get("existence", True), toml_machine.get("metadata", {}), + toml_machine.get("instance_id", "UNKNOWN_ID")) + usage_temp = machine.compute_server_info(requested_start_date, self.script_start) + server_usages.append(usage_temp) + + return { + "tenant_usage": { + "tenant_id": project.get("project_id", "UNKNOWN_ID"), + "server_usages": server_usages, + "start": start, + "stop": stop + } + } + + def get_server_details(self, project): + servers = [] + for toml_machine in project.get("machines", []): + machine = self.DummyMachine(toml_machine.get("cpus", 4), toml_machine.get("ram", 8), + toml_machine.get("existence", True), toml_machine.get("metadata", {}), + toml_machine.get("instance_id", "UNKNOWN_ID")) + temp_dict = machine.get_details() + servers.append(temp_dict) + return {"servers": servers} + + def get(self, url): + if not isinstance(url, str): + raise TypeError + if self.os_simple_tenant_usage_string in url: + request = url.split(self.os_simple_tenant_usage_string, 1)[1] + requested_project_id = request.split("?", 1)[0] + requested_start_date = request.split("start=", 1)[1] + for domain_name, domain_content in self.dummy_values.items(): + projects_in_domain = domain_content.get("projects", []) + for project_in_domain in projects_in_domain: + if project_in_domain.get("project_id", "UNKNOWN_ID") == requested_project_id: + tenant_usage = self.get_tenant_usage(project_in_domain, requested_start_date) + response = requests.Response() + response._content = json.dumps(tenant_usage, default=str).encode("utf-8") + return response + elif self.server_detail_all_tenants_string in url: + requested_project_id = url.split("=", 2)[2] + for domain_name, domain_content in self.dummy_values.items(): + projects_in_domain = domain_content.get("projects", []) + for project_in_domain in projects_in_domain: + if project_in_domain.get("project_id", "UNKNOWN_ID") == requested_project_id: + server_details = self.get_server_details(project_in_domain) + response = requests.Response() + response._content = json.dumps(server_details, default=str).encode("utf-8") + return response + + response = requests.Response() + response._content = b'{}' + return response diff --git a/project_usage_exporter.py b/project_usage_exporter.py index a384f6f..4abac3b 100755 --- a/project_usage_exporter.py +++ b/project_usage_exporter.py @@ -4,7 +4,7 @@ format. Alternatively develop in local mode and emulate machines and projects. """ - +import json from argparse import ( ArgumentParser, ArgumentDefaultsHelpFormatter, @@ -16,28 +16,23 @@ Optional, TextIO, Set, - Tuple, Dict, - List, - NamedTuple, - Union, - cast, Iterable, ) -from json import load from time import sleep -from urllib import parse from datetime import datetime, timedelta from os import getenv from dataclasses import dataclass from hashlib import sha256 as sha256func -from enum import Enum + +import toml + +from dummy_cloud import DummyCloud import openstack # type: ignore import prometheus_client # type: ignore import keystoneauth1 # type: ignore import maya -import toml import ast import requests @@ -81,13 +76,11 @@ dummy_file_env_var = "USAGE_EXPORTER_DUMMY_FILE" -default_dummy_file = "resources/dummy_machines.toml" +dummy_weights_file_env_var = "USAGE_EXPORTER_DUMMY_WEIGHTS_FILE" -UsageTuple = NamedTuple("UsageTuple", [("vcpu_hours", float), ("mb_hours", float)]) +default_dummy_file = "resources/dummy_cc.toml" -hour_timedelta = timedelta(hours=1) - -script_start = datetime.now() +default_dummy_weights_file = "resources/dummy_weights.toml" def sha256(content: str) -> str: @@ -119,7 +112,8 @@ def __init__( vcpu_weights=None, mb_weights=None, simple_vm_project="", - simple_vm_tag=None + simple_vm_tag=None, + dummy_file: TextIO = None ) -> None: self.domains = set(domains) if domains else None self.domain_id = domain_id @@ -130,25 +124,30 @@ def __init__( self.mb_weights = mb_weights self.simple_vm_project = simple_vm_project self.simple_vm_tag = simple_vm_tag + self.dummy_file = dummy_file if vcpu_weights is not None and mb_weights is not None: self.weights = {0: {"memory_mb": mb_weights, "vcpus": vcpu_weights}} else: self.weights = None - try: - self.cloud = openstack.connect() - except keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions: - logging.exception( - "Could not authenticate against OpenStack, Aborting! " - "See following traceback." - ) - logging.info("Consider using the dummy mode for testing") - raise ValueError + if self.dummy_file is not None: + self.cloud = DummyCloud(self.dummy_file, self.stats_start) + else: + try: + self.cloud = openstack.connect() + except keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions: + logging.exception( + "Could not authenticate against OpenStack, Aborting! " + "See following traceback." + ) + logging.info("Consider using the dummy mode for testing") + raise ValueError self.update() def update(self) -> None: self.projects = self.collect_projects() + print(self.projects) self.usages = self.collect_usages( - start=self.stats_start.strftime("%Y-%m-%dT%H:%M:%S") + start=self.stats_start.strftime("%Y-%m-%dT%H:%M:%S.%f") ) self.set_metrics() @@ -191,15 +190,15 @@ def collect_usages(self, **query_args) -> Dict[OpenstackProject, Dict[str, float "Received following invalid json payload: %s", json_payload ) continue - except BaseException: - logging.exception("Received following exception:") + except BaseException as e: + logging.exception(f"Received following exception:\n{e}") continue if project.is_simple_vm_project: if self.simple_vm_tag is None: logging.error("The simple vm tag is not set, please set the simple vm metadata tag for simple vm tracking") else: json_payload_metadata = self.cloud.compute.get( # type: ignore - f"/servers/detail?all_tenants=true&project_id=" + project.id + f"/servers/detail?all_tenants=false&project_id=" + project.id ).json() instance_id_to_project_dict = {} for instance in json_payload_metadata['servers']: @@ -236,7 +235,6 @@ def collect_usages(self, **query_args) -> Dict[OpenstackProject, Dict[str, float metric_amount = instance[instance_metric] total_usage += (instance_hours * metric_amount) * self.get_instance_weight(instance_metric, metric_amount, instance["started_at"]) project_usages[project][metric] = total_usage - return project_usages def get_instance_weight(self, metric_tag, metric_amount, started_date): @@ -252,7 +250,11 @@ def get_instance_weight(self, metric_tag, metric_amount, started_date): if associated_weights is not None: metric_weights = associated_weights[metric_tag] sorted_keys = sorted(metric_weights.keys()) - max_key = max(sorted_keys) + try: + max_key = max(sorted_keys) + except ValueError as e: + logging.exception(e) + return 1 for key in sorted_keys: if metric_amount <= key or max_key == key: return metric_weights[key] @@ -290,7 +292,7 @@ def collect_projects(self) -> Set[OpenstackProject]: return projects -def add_project(id, name, domain_name, domain_id, simple_vm_id, projects): +def add_project(id, name, domain_id, domain_name, simple_vm_id, projects): is_simple_vm_project = False if id == simple_vm_id: is_simple_vm_project = True @@ -305,174 +307,6 @@ def add_project(id, name, domain_name, domain_id, simple_vm_id, projects): ) -class ExistenceInformation(Enum): - NO_EXISTENCE = 0 - SINCE_SCRIPT_START = 1 - SINCE_DATETIME = 2 - BETWEEN_DATETIMES = 3 - - -@dataclass -class DummyMachine: - """ - Representing a dummy machine causing usage to monitor. - :param name: Currently not used outside but might be in future, therefore leave it - :param cpus: Number of cpus the dummy machine is using. - :param ram: Amount of RAM [GiB] the machine is using. - :param existence: Determines whether the machine is *up* and its usage so far. In case - of True the machine is considered booted up the instant this script is started. In - case of False it hasn't been booted ever (no actual use case). - In case of a single datetime the machine is considered *up* since that moment (for - simplicity the timezone information are ignored). In case of a list of two datetimes - the machine is considered *up* the time in between. The first one must be - older/smaller than the second one and both but relative to the moment the script - started both may lie in the future or past. - """ - - cpus: int = 4 - ram: int = 8 - existence: Union[bool, datetime, Tuple[datetime, datetime]] = True - - def __post_init__(self) -> None: - if self.cpus <= 0 or self.ram <= 0: - raise ValueError("`cpu` and `ram` must be positive") - if isinstance(self.existence, (list, tuple)): - if self.existence[0] > self.existence[1]: # type: ignore - raise ValueError( - "First existence-tuple datetime must be older than second one" - ) - # remove any timezone information - self.existence_information = ExistenceInformation.BETWEEN_DATETIMES - elif isinstance(self.existence, datetime): - self.existence_information = ExistenceInformation.SINCE_DATETIME - elif isinstance(self.existence, bool): - self.existence_information = ( - ExistenceInformation.SINCE_SCRIPT_START - if self.existence - else ExistenceInformation.NO_EXISTENCE - ) - else: - raise ValueError( - f"Invalid type for param `existence` (got {type(self.existence)}" - ) - - @property - def ram_mb(self) -> int: - return self.ram * 1024 - - def usage_value(self) -> UsageTuple: - """ - Returns the total ram and cpu usage counted in hours of this machine, depending - on its `existence` configuration` - """ - now = datetime.now() - if self.existence_information is ExistenceInformation.NO_EXISTENCE: - return UsageTuple(0, 0) - elif self.existence_information is ExistenceInformation.SINCE_SCRIPT_START: - hours_existence = (datetime.now() - script_start) / hour_timedelta - return UsageTuple( - self.cpus * hours_existence, self.ram_mb * hours_existence - ) - elif self.existence_information is ExistenceInformation.SINCE_DATETIME: - # to satisfy `mypy` type checker - boot_datetime = cast(datetime, self.existence) - hours_existence = ( - now - boot_datetime.replace(tzinfo=None) - ) / hour_timedelta - # do not report negative usage in case the machine is not *booted yet* - if hours_existence > 0: - return UsageTuple( - self.cpus * hours_existence, self.ram_mb * hours_existence - ) - else: - return UsageTuple(0, 0) - else: - # to satisfy `mypy` type checker - runtime_tuple = cast(Tuple[datetime, datetime], self.existence) - boot_datetime = cast(datetime, runtime_tuple[0].replace(tzinfo=None)) - shutdown_datetime = cast(datetime, runtime_tuple[1].replace(tzinfo=None)) - if boot_datetime > now: - # machine did not boot yet - return UsageTuple(0, 0) - elif shutdown_datetime < now: - # machine did run already and is considered down - hours_existence = (shutdown_datetime - boot_datetime) / hour_timedelta - else: - # machine booted in the past but is still existing - hours_existence = (now - boot_datetime) / hour_timedelta - return UsageTuple( - self.cpus * hours_existence, self.ram_mb * hours_existence - ) - - -class DummyExporter(_ExporterBase): - def __init__( - self, - dummy_values: TextIO, - domains: Iterable[str] = None, - domain_id: Optional[str] = None, - ) -> None: - self.dummy_values = toml.loads(dummy_values.read()) - self.domains = set(domains) if domains else None - self.domain_id = domain_id - self.projects: List[DummyProject] = [] - for project_name, project_content in self.dummy_values.items(): - machines = [ - DummyMachine(**machine) for machine in project_content["machines"] - ] - self.projects.append( - DummyProject( - name=project_name, - domain_name=project_content.get("domain", ""), - machines=machines, - ) - ) - self.update() - - def update(self) -> None: - for project in self.projects: - if self.domain_id and project.domain_id != self.domain_id: - logging.info( - "Skipping exporting project %s since its domain id " - "is not requested", - project, - ) - continue - if self.domains and project.domain_name not in self.domains: - logging.info( - "Skipping exporting project %s since its domain " - "is not requested", - project, - ) - continue - project_usages = [machine.usage_value() for machine in project.machines] - vcpu_hours = sum(usage.vcpu_hours for usage in project_usages) - mb_hours = sum(usage.mb_hours for usage in project_usages) - project_metrics["total_vcpus_usage"].labels( - project_id=project.id, - project_name=project.name, - domain_name=project.domain_name, - domain_id=project.domain_id, - ).set(vcpu_hours) - project_metrics["total_memory_mb_usage"].labels( - project_id=project.id, - project_name=project.name, - domain_name=project.domain_name, - domain_id=project.domain_id, - ).set(mb_hours) - - -@dataclass -class DummyProject: - name: str - machines: List[DummyMachine] - domain_name: str = "" - - def __post_init__(self): - self.id = sha256(self.name)[-16:] - self.domain_id = sha256(self.domain_name)[-16:] - - def valid_date(s): try: return maya.when(s).datetime() @@ -481,6 +315,13 @@ def valid_date(s): raise ArgumentTypeError(msg) +def get_dummy_weights(file): + dummy_weights = toml.loads(file.read()) + response = requests.Response() + response._content = json.dumps(dummy_weights["weights"], default=str).encode("utf-8") + return response + + def main(): parser = ArgumentParser( epilog=f"{__license__} @ {__author__}", @@ -492,10 +333,19 @@ def main(): "--dummy-data", type=FileType(), help=f"""Use dummy values instead of connecting to an openstack instance. Usage - values are calculated base on the configured existence, take a look at the - example file for an explanation {default_dummy_file}. Can also be provided via + values are calculated based on the configured existence. Toml files can be updated on the fly as they are read + every time a dummy-cloud function is called (functions of nested classes excluded). + Take a look at the example file for an explanation {default_dummy_file}. Can also be provided via environment variable ${dummy_file_env_var}""", ) + parser.add_argument( + "-w", + "--dummy-weights", + type=FileType(), + help=f"""Use dummy weight endpoint instead of connecting to the api. Take a look at the + example file for an explanation {default_dummy_weights_file}. Can also be provided via + environment variable ${dummy_weights_file_env_var}""", + ) parser.add_argument( "--domain", default=[ @@ -526,7 +376,7 @@ def main(): help=f"""Use weights for different numbers of cpus in a vm. Value is given as the string representation of a dictionary with ints as keys and as values. a weight of 1 means no change. Above 1 its more expensive, under one it is less - expensive. Not available with dummy mode. Can also be set via ${vcpu_weights_env_var}""", + expensive. Can also be set via ${vcpu_weights_env_var}""", ) parser.add_argument( "--mb-weights", @@ -535,7 +385,7 @@ def main(): help=f"""Use weights for different numbers of mb (of ram) in a vm. Value is given as the string representation of a dictionary with ints as keys and as values. a weight of 1 means no change. Above 1 its more expensive, under one it is less - expensive. Not available with dummy mode. Can also be set via ${project_mb_weights_env_var}""", + expensive. Can also be set via ${project_mb_weights_env_var}""", ) parser.add_argument( "--simple-vm-id", @@ -589,13 +439,27 @@ def main(): args = parser.parse_args() if args.dummy_data: logging.info("Using dummy export with data from %s", args.dummy_data.name) - exporter = DummyExporter(args.dummy_data, args.domain, args.domain_id) + try: + exporter = OpenstackExporter( + domains=args.domain, stats_start=args.start, domain_id=args.domain_id, + vcpu_weights=ast.literal_eval(args.vcpu_weights), mb_weights=ast.literal_eval(args.mb_weights), + simple_vm_project=args.simple_vm_id, simple_vm_tag=args.simple_vm_tag, dummy_file=args.dummy_data + ) + except ValueError as e: + return 1 elif getenv(dummy_file_env_var): logging.info("Using dummy export with data from %s", getenv(dummy_file_env_var)) # if the default dummy data have been used we need to open them, argparse # hasn't done this for us since the default value has not been a string - with open(getenv(dummy_file_env_var)) as file: - exporter = DummyExporter(file, args.domain, args.domain_id) + try: + with open(getenv(dummy_file_env_var)) as file: + exporter = OpenstackExporter( + domains=args.domain, stats_start=args.start, domain_id=args.domain_id, + vcpu_weights=ast.literal_eval(args.vcpu_weights), mb_weights=ast.literal_eval(args.mb_weights), + simple_vm_project=args.simple_vm_id, simple_vm_tag=args.simple_vm_tag, dummy_file=file + ) + except ValueError as e: + return 1 else: try: logging.info("Using regular openstack exporter") @@ -609,10 +473,18 @@ def main(): logging.info(f"Beginning to serve metrics on port {args.port}") prometheus_client.start_http_server(args.port) laps = args.weight_update_frequency + if args.dummy_weights or getenv(dummy_weights_file_env_var): + args.weight_update_endpoint = "dummy-endpoint" while True: if laps >= args.weight_update_frequency and args.weight_update_endpoint != "": try: - weight_response = requests.get(args.weight_update_endpoint) + if args.dummy_weights: + weight_response = get_dummy_weights(args.dummy_weights) + elif getenv(dummy_weights_file_env_var): + with open(getenv(dummy_weights_file_env_var)) as file: + weight_response = get_dummy_weights(file) + else: + weight_response = requests.get(args.weight_update_endpoint) current_weights = { x['resource_set_timestamp']: {'memory_mb': {y['value']: y['weight'] for y in x['memory_mb']}, 'vcpus': {y['value']: y['weight'] for y in x['vcpus']}} for diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..44721b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,38 @@ +appdirs==1.4.3 +asn1crypto==0.24.0 +certifi==2019.3.9 +cffi==1.12.3 +chardet==3.0.4 +cryptography==2.6.1 +dateparser==0.7.1 +decorator==4.4.1 +dogpile.cache==0.7.1 +humanize==0.5.1 +idna==2.8 +iso8601==0.1.12 +jmespath==0.9.4 +jsonpatch==1.23 +jsonpointer==2.0 +keystoneauth1==3.18.0 +maya==0.6.1 +munch==2.3.2 +netifaces==0.10.9 +openstacksdk==0.48.0 +os-service-types==1.7.0 +pbr==5.2.0 +pendulum==2.0.4 +prometheus-client==0.6.0 +pycparser==2.19 +python-dateutil==2.8.0 +pytz==2019.1 +pytzdata==2019.1 +pyyaml==5.1 +regex==2019.4.14 +requests==2.21.0 +requestsexceptions==1.4.0 +six==1.12.0 +snaptime==0.2.4 +stevedore==2.0.1 +toml==0.10.0 +tzlocal==2.0.0b1 +urllib3==1.24.3 diff --git a/resources/credits_machines.toml b/resources/credits_machines.toml deleted file mode 100644 index 68f49d7..0000000 --- a/resources/credits_machines.toml +++ /dev/null @@ -1,29 +0,0 @@ -# See github.com/toml-lang/toml for a configuration guide -# enter the project name in square brackets -# project_id will be the last 16 chars of its SHA256 -[credits] -# domain_id will be the last 16 chars of its SHA256 -domain = 'elixir' -machines = [ - # minimal information, this machine will be considered started the moment the - # information are parsed - {ram = 128, cpus = 24}, - # machine will be considered booted at the given time, may be in the future - {ram = 256, cpus = 20, existence = true}, - # considered online in between the two given timestamps - {ram = 32, cpus = 15, existence = true}, -] - -[credits_2] -domain = 'elixir' -machines = [ - {ram=2, cpus=8, existence=true}, - {ram=4, cpus=8, existence=true}, -] - -[credits_3] -domain = 'elixir' -machines = [ - {ram = 8, cpus = 16}, - {ram = 128, cpus = 16} -] diff --git a/resources/dummy_cc.toml b/resources/dummy_cc.toml new file mode 100644 index 0000000..5c738b3 --- /dev/null +++ b/resources/dummy_cc.toml @@ -0,0 +1,86 @@ +# See github.com/toml-lang/toml for a configuration guide +# For more information about toml files beside github see here https://learnxinyminutes.com/docs/toml/ +# +# Here: default domain for OS projects without vm metadata. +[default] # domain name +domain_id = "default" # domain id + +# creates a projects: [{project_name:somename, project_id:someid, machines=[]}] entry with data below +# use this format if none of the values inside a machine in machines is a dictionary +[[default.projects]] +project_name="MyOSP" +project_id = '123MyOSP' +machines = [ + # when existence = true, the machine is treated as if it was active since start-string (if start-string provided) + # or time of script-start (if start-string not provided) + {ram = 8, cpus = 2, existence = true, instance_id = "1"}, + # when existence = false, the machines is treated as if it was never active + {ram = 8, cpus = 4, existence = false, instance_id = "2"}, + # when existence = datetime, the machines is treated as if it was active since start-string (if start-string + # provided and start-string > datetime) or since datetime (if start_string not provided or start_string < datetime) + {ram = 8, cpus = 2, existence = 2020-07-15T14:30:00, instance_id = "3"}, + # when existence = [datetime_start, datetime_stop], the machines is treated as if it was active since start-string (if start-string + # provided and start-string > datetime_start) or since datetime (if start_string not provided or start_string < datetime_start) + # until datetime_stop + {ram = 8, cpus = 4, existence = [2020-07-15T14:30:00, 2020-07-16T20:40:00], instance_id = "4"}, +] + +# creates another project-json in projects array with data below +# { ..., projects = [{project_name=...}, {project_name="HerOSP", ...}] } +[[default.projects]] +project_name="HerOSP" +project_id = '123HerOSP' +machines = [ + {ram = 8, cpus = 2, existence = true, instance_id = "5"}, + {ram = 8, cpus = 4, existence = true, instance_id = "6"}, +] + +# Here: elixir domain for simplevm pool-projects with vm metadata. +[elixir] +domain_id="123elixirDomain" + +[[elixir.projects]] +project_name="portal-pool-dev" +project_id="123devsimplevm" + + # creates a {..., machines = [{ram=,cpus=,...}, {ram=,cpus=,...}], ...} entry with data below + # you have to use this format if a value inside a machine in machines is a dictionary (e.g. the metadata entry) + # otherwise you will encounter an inline error + [[elixir.projects.machines]] + ram = 8 + cpus = 2 + existence = true + metadata = {project_name = 'MySVM', project_id = ''} + instance_id = "7" + + [[elixir.projects.machines]] + ram = 8 + cpus = 4 + existence = true + metadata = {project_name = 'MySVM', project_id = ''} + instance_id = "8" + +[[elixir.projects]] +project_name="portal-pool" +project_id="123realsimplevm" + + [[elixir.projects.machines]] + ram = 8 + cpus = 4 + existence = true + metadata = {project_name = 'HerSVM', project_id = ''} + instance_id = "9" + + [[elixir.projects.machines]] + ram = 8 + cpus = 4 + existence = [2020-07-15T14:30:00, 2020-07-16T20:40:00] + metadata = {project_name = 'HisSVM', project_id = ''} + instance_id = "10" + + [[elixir.projects.machines]] + ram = 80 + cpus = 40 + existence = 2020-07-15T14:30:00 + metadata = {project_name = 'HisSVM', project_id = ''} + instance_id = "11" diff --git a/resources/dummy_machines.toml b/resources/dummy_machines.toml deleted file mode 100644 index a2f6c15..0000000 --- a/resources/dummy_machines.toml +++ /dev/null @@ -1,32 +0,0 @@ -# See github.com/toml-lang/toml for a configuration guide -# enter the project name in square brackets -# project_id will be the last 16 chars of its SHA256 -[admin] -# domain_id will be the last 16 chars of its SHA256 -domain = 'domainA' -machines = [ - # minimal information, this machine will be considered started the moment the - # information are parsed - {ram = 1, cpus = 2}, - # machine will be considered booted at the given time, may be in the future - {ram = 2, cpus = 2, existence = 2018-11-26T14:30:00Z}, - # this machine is considered offline, will not be calculated - # they are currently ignored, yet remain in case of changes - {ram = 2, cpus = 2, existence = false}, - # considered online in between the two given timestamps - {ram = 8, cpus = 24, existence = [2018-11-27T14:00:00Z, 2018-11-27T15:00:10Z]}, -] - -[demo] -domain = 'domainB' -machines = [ - {ram=2, cpus=8, existence=true}, - {ram=4, cpus=8, existence=true}, -] - -[TestProject] -domain = 'elixir' -machines = [ - {ram = 8, cpus = 16}, - {ram = 128, cpus = 16} -] diff --git a/resources/dummy_weights.toml b/resources/dummy_weights.toml new file mode 100644 index 0000000..0cafd9f --- /dev/null +++ b/resources/dummy_weights.toml @@ -0,0 +1,19 @@ +[[weights]] +resource_set_timestamp = 1594771200 # 2020-07-15T00:00:00+00:00 in ISO 8601 +vcpus = [ + {value = 2, weight = 1}, + {value = 4, weight = 2}, +] +memory_mb = [ + {value = 8192, weight = 2}, +] + +[[weights]] +resource_set_timestamp = 1594339200 # 2020-07-10T00:00:00+00:00 in ISO 8601 +vcpus = [ + {value = 2, weight = 2}, + {value = 4, weight = 3}, +] +memory_mb = [ + {value = 8192, weight = 3}, +] diff --git a/resources/high_usage.toml b/resources/high_usage.toml deleted file mode 100644 index a850c36..0000000 --- a/resources/high_usage.toml +++ /dev/null @@ -1,16 +0,0 @@ -# See github.com/toml-lang/toml for a configuration guide -# enter the project name in square brackets -# project_id will be its SHA256 -[ProjectA] -domain = 'elixir' -# considered online in between the two given timestamps -machines = [ - {ram = 8, cpus = 8, existence = true}, - {ram = 8, cpus = 6, existence = [2018-11-27T14:00:00Z, 2018-12-27T15:00:10Z]}, -] - -[credits] -domain = 'elixir' -machines = [ - {ram=32, cpus=24, existence True}, -] diff --git a/resources/low_usage.toml b/resources/low_usage.toml deleted file mode 100644 index 1b69eb0..0000000 --- a/resources/low_usage.toml +++ /dev/null @@ -1,16 +0,0 @@ -# See github.com/toml-lang/toml for a configuration guide -# enter the project name in square brackets -# project_id will be its SHA256 -[DataTransfer] -domain = 'DomainA' -# minimal information, this machine will be considered started the moment the -# information are parsed -machines = [ - {ram=2, cpus=2}, -] - -[PingTest] -domain = 'elixir' -machines = [ - {ram = 1, cpus = 1}, -] diff --git a/test_os_outputs.py b/test_os_outputs.py new file mode 100755 index 0000000..8d181e0 --- /dev/null +++ b/test_os_outputs.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +import os +import json + +import openstack # type: ignore +import logging + + +class OSOutputTester: + def __init__( + self, + ) -> None: + self.USERNAME = os.environ["OS_USERNAME"] + self.PASSWORD = os.environ["OS_PASSWORD"] + self.PROJECT_NAME = os.environ["OS_PROJECT_NAME"] + self.USER_DOMAIN_NAME = os.environ["OS_USER_DOMAIN_NAME"] + self.AUTH_URL = os.environ["OS_AUTH_URL"] + self.PROJECT_DOMAIN_ID = os.environ["OS_PROJECT_DOMAIN_ID"] + self.REGION_NAME = os.environ["OS_REGION_NAME"] + self.INTERFACE = os.environ["OS_INTERFACE"] + self.IDENTITDY = os.environ["OS_IDENTITY_API_VERSION"] + self.PROJECT_ID = os.environ["OS_PROJECT_ID"] + + try: + self.cloud = openstack.connection.Connection( + username=self.USERNAME, + password=self.PASSWORD, + auth_url=self.AUTH_URL, + project_name=self.PROJECT_NAME, + user_domain_name=self.USER_DOMAIN_NAME, + project_domain_id=self.PROJECT_DOMAIN_ID, + region_name=self.REGION_NAME, + identity_interface = self.INTERFACE + ) + self.cloud.authorize() + except Exception as e: + print("Could not authenticate against OpenStack, Aborting! " + "See following traceback.") + print(e) + raise ValueError + print("Connected to Openstack!") + + def list_projects(self): + try: + print("------PRINTING ALL PROJECTS---------------------------------------------------------") + projects = self.cloud.list_projects() + print(projects) + print(len(projects)) + print() + except Exception as e: + print("Could not load all projects:") + logging.exception(e) + print() + try: + print("------PRINTING PROJECTS FOR DOMAIN_ID {0}--------------------------------------------------------".format(self.PROJECT_DOMAIN_ID)) + projects_two = self.cloud.list_projects(domain_id=self.PROJECT_DOMAIN_ID) + print(projects_two) + print(len(projects_two)) + print() + except Exception as e: + print("Could not load projects with domain id") + logging.exception(e) + print() + + def get_domain(self): + try: + print("------PRINTING DOMAIN------------------------------------------------------------------------------") + domain = self.cloud.get_domain(name_or_id=self.USER_DOMAIN_NAME) + print(json.dumps(domain, indent=2)) + print() + except Exception as e: + print("Could not get domain") + logging.exception(e) + print() + + def compute_get(self): + try: + print("------PRINTING COMPUTE GET OS SIMPLE TENANT USAGE--------------------------------------------------") + payload = self.cloud.compute.get( # type: ignore + f"/os-simple-tenant-usage/{self.PROJECT_ID}?start=2020-07-15T15:07:51.211724" + ).json() + print(json.dumps(payload, indent=2)) + print() + except Exception as e: + print("Could not compute") + logging.exception(e) + print() + try: + print("------PRINTING COMPUTE GET SERVER DETAILS----------------------------------------------------------") + payload_two = self.cloud.compute.get( # type: ignore + f"/servers/detail?all_tenants=false&project_id=" + self.PROJECT_ID + ).json() + print(json.dumps(payload_two, indent=2)) + print(len(payload_two["servers"])) + except Exception as e: + print("Could not compute the second") + logging.exception(e) + print() + + +def main(): + dummy_cloud_tester = OSOutputTester() + dummy_cloud_tester.list_projects() + dummy_cloud_tester.get_domain() + dummy_cloud_tester.compute_get() + + +if __name__ == "__main__": + + exit(main()) From 2a3d832ebe3beea278c00a81974d8af432ee5c11 Mon Sep 17 00:00:00 2001 From: ekatchko Date: Tue, 21 Jul 2020 14:30:50 +0200 Subject: [PATCH 2/3] update --- .gitignore | 2 - .python-version | 1 + Dockerfile | 5 ++- README.md | 78 +++++++++++++++++++++++++++++++-------- dummy_cloud.py | 19 +++++++--- project_usage_exporter.py | 2 +- 6 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 .python-version diff --git a/.gitignore b/.gitignore index 84c6866..0bd856a 100644 --- a/.gitignore +++ b/.gitignore @@ -98,8 +98,6 @@ profile_default/ ipython_config.py # pyenv -.python-version - # celery beat schedule file celerybeat-schedule diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..a0fc9e0 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.7.8 diff --git a/Dockerfile b/Dockerfile index 6c20c2a..9e37e90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ FROM python:3.7-alpine RUN apk add --no-cache linux-headers musl-dev gcc libffi-dev openssl-dev -COPY . /code WORKDIR /code -RUN pip install --no-cache pipenv && pipenv install --system --deploy --ignore-pipfile +ADD requirements.txt /code +RUN pip install -r requirements.txt +COPY . /code EXPOSE 8080 CMD ["python", "project_usage_exporter.py"] diff --git a/README.md b/README.md index eabf69c..d8578a3 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,16 @@ Exported labels per project are: - `project_name` ## Requirements/Installation +All dependencies are managed with a requirements.txt. Create a virtual environment with a tool of your choosing +(e.g. [*pyenv*](https://github.com/pyenv/pyenv) with [*pyenv-virtualenv*](https://github.com/pyenv/pyenv-virtualenv) or [*venv*](https://docs.python.org/3/library/venv.html)) and +install with `pip install -r requirements.txt`. -All production and development dependencies are managed via +Deprecated: +~~All production and development dependencies are managed via [*pipenv*](https://pipenv.readthedocs.io). Therefore simply go via `pipenv install` or start directly with one of the modi listed below. You can activate the virtual environment via `pipenv shell` or simply prefix any command with `pipenv run` to have it -run inside the corresponding environment. +run inside the corresponding environment.~~ A [docker image](https://hub.docker.com/r/tluettje/os_project_usage_exporter/) is available as well and all command line options do have corresponding environment @@ -39,11 +43,20 @@ optional arguments: -h, --help show this help message and exit -d DUMMY_DATA, --dummy-data DUMMY_DATA Use dummy values instead of connecting to an openstack - instance. Usage values are calculated base on the - configured existence, take a look at the example file - for an explanation resources/dummy_machines.toml. Can - also be provided via environment variable - $USAGE_EXPORTER_DUMMY_FILE (default: None) + instance. Usage values are calculated based on the + configured existence. Toml files can be updated on the + fly as they are read every time a dummy-cloud function + is called (functions of nested classes excluded). Take + a look at the example file for an explanation + resources/dummy_cc.toml. Can also be provided via + environment variable $USAGE_EXPORTER_DUMMY_FILE + (default: None) + -w DUMMY_WEIGHTS, --dummy-weights DUMMY_WEIGHTS + Use dummy weight endpoint instead of connecting to the + api. Take a look at the example file for an + explanation resources/dummy_weights.toml. Can also be + provided via environment variable + $USAGE_EXPORTER_DUMMY_WEIGHTS_FILE (default: None) --domain [DOMAIN [DOMAIN ...]] Only export usages of projects belonging to one of the given domains. Separate them via comma if passing via @@ -55,17 +68,52 @@ optional arguments: identified by the given ID. Takes precedence over any specified domain and default values. Can also be set via $USAGE_EXPORTER_PROJECT_DOMAIN_ID (default: ) + --vcpu-weights VCPU_WEIGHTS + Use weights for different numbers of cpus in a vm. + Value is given as the string representation of a + dictionary with ints as keys and as values. a weight + of 1 means no change. Above 1 its more expensive, + under one it is less expensive. Can also be set via + $USAGE_EXPORTER_VCPU_WEIGHTS (default: {}) + --mb-weights MB_WEIGHTS + Use weights for different numbers of mb (of ram) in a + vm. Value is given as the string representation of a + dictionary with ints as keys and as values. a weight + of 1 means no change. Above 1 its more expensive, + under one it is less expensive. Can also be set via + $USAGE_EXPORTER_PROJECT_MB_WEIGHTS (default: {}) + --simple-vm-id SIMPLE_VM_ID + The ID of the Openstack project, that hosts the + SimpleVm projects. Can also be set vis + $USAGE_EXPORTER_SIMPLE_VM_PROJECT_ID (default: ) + --simple-vm-tag SIMPLE_VM_TAG + The metadata of the Openstack project, that hosts the + SimpleVm projects. It is used to differentiate the + simple vm projects, default: project_name Can also be + set vis $USAGE_EXPORTER_SIMPLE_VM_PROJECT_TAG + (default: project_name) + --weight-update-frequency WEIGHT_UPDATE_FREQUENCY + The frequency of checking if there is a weight update. + Is a multiple of the update interval length . Defaults + to the value of environment variable + $USAGE_EXPORTER_WEIGHT_UPDATE_FREQUENCY or 10 + (default: 10) + --weight-update-endpoint WEIGHT_UPDATE_ENDPOINT + The endpoint url where the current weights can be + updated . Defaults to the value of environment + variable $USAGE_EXPORTER_WEIGHTS_UPDATE_ENDPOINT or + will be left blank (default: ) -s START, --start START Beginning time of stats (YYYY-MM-DD). If set the value of environment variable $USAGE_EXPORTER_START_DATE is - used. Uses maya for parsing. (default: 2019-01-30 - 15:29:32.363451) + used. Uses maya for parsing. (default: 2020-07-21 + 14:24:34.159480) -i UPDATE_INTERVAL, --update-interval UPDATE_INTERVAL Time to sleep between intervals, in case the calls cause to much load on your openstack instance. Defaults to the value of environment variable $USAGE_EXPORTER_UPDATE_INTERVAL or 300 (in seconds) - (default: 300) + (default: 30) -p PORT, --port PORT Port to provide metrics on (default: 8080) GNU AGPLv3 @ tluettje @@ -78,14 +126,14 @@ access to an OpenStack instance you can emulate running projects and machines wi simple `toml` files. A few profiles are available inside the `/resources` folder. ```shell -pipenv run ./project_usage_exporter.py \ - --dummy-data resources/dummy_machines.toml \ - --update-interval 10 --domain +./project_usage_exporter.py \ + -d resources/dummy_cc.toml -w resources/dummy_weights.toml \ + --vcpu-weights "{2:1}" --mb-weights "{1024:1}" --domain ``` or ``` -docker run -e USAGE_EXPORTER_DUMMY_FILE=/code/resources/dummy_machines.toml \ - -e USAGE_EXPORTER_UPDATE_INTERVAL=10 \ +docker run -e USAGE_EXPORTER_DUMMY_FILE=/code/resources/dummy_cc.toml \ + -e USAGE_EXPORTER_DUMMY_WEIGHTS_FILE=/code/resources/dummy_weigths.toml \ -e USAGE_EXPORTER_PROJECT_DOMAINS= \ -p 8080:8080 tluettje/os_project_usage_exporter:v2 ``` diff --git a/dummy_cloud.py b/dummy_cloud.py index 63eee4d..4dcee1a 100644 --- a/dummy_cloud.py +++ b/dummy_cloud.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta - +from os import getenv import toml from munch import Munch import requests @@ -8,10 +8,12 @@ Union, cast, ) +import logging from enum import Enum import json hour_timedelta = timedelta(hours=1) +dummy_file_env_var = "USAGE_EXPORTER_DUMMY_FILE" class DummyCloud: @@ -20,15 +22,21 @@ def __init__(self, dummy_file, start=None): self.dummy_file = dummy_file self.dummy_values = toml.loads(self.dummy_file.read()) if start is not None: - script_start = start + script_start = start.replace(tzinfo=None) else: script_start = datetime.now() self.compute = Compute(self.dummy_values, script_start) def load_toml(self): - self.dummy_file.seek(0) - self.dummy_values = toml.loads(self.dummy_file.read()) - self.compute.reload(self.dummy_values) + try: + with open(getenv(dummy_file_env_var)) as file: + file.seek(0) + self.dummy_values = toml.loads(file.read()) + self.compute.reload(self.dummy_values) + except: + self.dummy_file.seek(0) + self.dummy_values = toml.loads(self.dummy_file.read()) + self.compute.reload(self.dummy_values) def list_projects(self, domain_id=None): self.load_toml() @@ -41,6 +49,7 @@ def list_projects(self, domain_id=None): project = Munch() project.id = project_in_domain.get("project_id", "UNKNOWN_ID") project.name = project_in_domain.get("project_name", "UNKNOWN_NAME") + project.domain_id = domain_content.get("domain_id", "UNKNOWN_DOMAIN_ID") projects_return.append(project) return projects_return diff --git a/project_usage_exporter.py b/project_usage_exporter.py index 4abac3b..02301b7 100755 --- a/project_usage_exporter.py +++ b/project_usage_exporter.py @@ -428,7 +428,7 @@ def main(): "-i", "--update-interval", type=int, - default=int(getenv(update_interval_env_var, 300)), + default=int(getenv(update_interval_env_var, 30)), help=f"""Time to sleep between intervals, in case the calls cause to much load on your openstack instance. Defaults to the value of environment variable ${update_interval_env_var} or 300 (in seconds)""", From 4f857851b87ba5821b3541f686787c03ca214683 Mon Sep 17 00:00:00 2001 From: ekatchko Date: Thu, 23 Jul 2020 18:59:12 +0200 Subject: [PATCH 3/3] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8578a3..440b472 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ simple `toml` files. A few profiles are available inside the `/resources` folder ```shell ./project_usage_exporter.py \ -d resources/dummy_cc.toml -w resources/dummy_weights.toml \ - --vcpu-weights "{2:1}" --mb-weights "{1024:1}" --domain + --vcpu-weights "{2:1}" --mb-weights "{1024:1}" --domain --simple-vm-id 123realsimplevm ``` or ```