Skip to content

Commit

Permalink
Pull request update/240716
Browse files Browse the repository at this point in the history 
7077f2c OS-3904. Fix resource type for RDD resources detected as volumes
c880339 OS-4883. Rightsizing RDS archive recommendation
  • Loading branch information
@stanfra
stanfra authored Jul 16, 2024
2 parents 48bb5fe + 7077f2c commit 4faea0a
Show file tree
Hide file tree
Showing 5 changed files with 289 additions and 170 deletions.
232 changes: 84 additions & 148 deletions bumiworker/bumiworker/modules/archive/rightsizing_instances.py
View file
Original file line number Diff line number Diff line change
@@ -1,175 +1,111 @@
import logging

from collections import defaultdict
from datetime import datetime, timedelta

from bumiworker.bumiworker.consts import ArchiveReason
from bumiworker.bumiworker.modules.base import ArchiveBase
from bumiworker.bumiworker.modules.recommendations.rightsizing_instances import (
RightsizingInstances as RightsizingInstancesRecommendation)
RightsizingInstances as RightsizingInstancesRecommendation
)
from bumiworker.bumiworker.modules.rightsizing_base import (
HOURS_IN_DAY, DAYS_IN_MONTH)
HOURS_IN_DAY, DAYS_IN_MONTH, RightsizingArchiveBase)

LOG = logging.getLogger(__name__)
NEBIUS_CLOUD_TYPE = 'nebius'


class RightsizingInstances(ArchiveBase, RightsizingInstancesRecommendation):
class RightsizingInstances(RightsizingArchiveBase,
RightsizingInstancesRecommendation):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.reason_description_map[ArchiveReason.RECOMMENDATION_APPLIED] = (
'flavor changed')
self.reason_description_map[ArchiveReason.RECOMMENDATION_IRRELEVANT] = (
self.reason_description_map[
ArchiveReason.RECOMMENDATION_IRRELEVANT] = (
'recommended flavor more expensive')

@property
def supported_cloud_types(self):
return list(self._get_supported_func_map().keys())

def _get_instances(self, cloud_account_ids, start_date):
instances = super()._get_instances(cloud_account_ids, start_date)
instances_by_account_map = {x: [] for x in cloud_account_ids}
for instance in instances:
cloud_account_id = instance['cloud_account_id']
instances_by_account_map[cloud_account_id].append(instance)
return instances_by_account_map

def _get(self, previous_options, optimizations, cloud_accounts_map,
**kwargs):
days_threshold = previous_options['days_threshold']

cloud_acc_instances_map = defaultdict(dict)
min_dt = datetime.utcnow() - timedelta(days=days_threshold)
for cloud_acc_id, instances in self._get_instances(
list(cloud_accounts_map.keys()), int(min_dt.timestamp())).items():
for instance in instances:
instance_key = self.get_record_key(instance)
cloud_acc_instances_map[cloud_acc_id][instance_key] = instance

account_optimizations_map = defaultdict(dict)
for optimization in optimizations:
optimization_key = self.get_record_key(optimization)
account_optimizations_map[optimization['cloud_account_id']][
optimization_key] = optimization

result = []
for cloud_account_id, optimizations_dict in account_optimizations_map.items():
cloud_account = cloud_accounts_map.get(cloud_account_id)
if not cloud_account:
for optimization in optimizations_dict.values():
def set_additional_reasons(self, cloud_resource_map, cloud_account,
cloud_resource_id_instance_map,
optimizations_dict, days_threshold, result):
cloud_resource_id_info_map = self._get_instances_info(
list(cloud_resource_map.values()), cloud_account['id'],
cloud_account['type'])
metrics_map = self._get_metrics(
list(cloud_resource_map.keys()), cloud_account['id'],
days_threshold, cloud_account['type'])
current_flavor_params = self._get_flavor_params(
cloud_resource_id_info_map, cloud_resource_id_instance_map,
cloud_account['type'])

for params in current_flavor_params:
cloud_resource_ids = params['resource_ids']
region = params['region']
family_specs = params['family_specs']
flavor_params = params['flavor_params']
if cloud_account['type'] == NEBIUS_CLOUD_TYPE:
flavor_params['cloud_account_id'] = cloud_account['id'],
current_flavor = self._find_flavor(
cloud_account['type'], region, family_specs, 'current',
**flavor_params)
if not current_flavor:
for cloud_resource_id in cloud_resource_ids:
instance = cloud_resource_id_instance_map[
cloud_resource_id]
instance_key = self.get_record_key(instance)
optimization = optimizations_dict[instance_key]
self._set_reason_properties(
optimization, ArchiveReason.CLOUD_ACCOUNT_DELETED)
optimization, ArchiveReason.FAILED_DEPENDENCY,
'flavor not found')
result.append(optimization)
continue

instances_map = cloud_acc_instances_map.get(cloud_account_id, {})
cloud_resource_map = {}
cloud_resource_id_instance_map = {}
for instance_key, optimization in optimizations_dict.items():
instance = instances_map.get(instance_key)
meta = instance['meta']
inst_flavor = meta.get('flavor') or meta.get('platform_name')
if not instance:
self._set_reason_properties(
optimization, ArchiveReason.RESOURCE_DELETED)
result.append(optimization)
continue
elif (inst_flavor == optimization['recommended_flavor'] and
meta.get('cpu_count') == optimization['recommended_flavor_cpu']):
cpu_flavor_map = {}
for cloud_resource_id in cloud_resource_ids:
current_r_info = cloud_resource_id_info_map[
cloud_resource_id][0]
if (cloud_account['type'] == 'azure_cnr' and len(
cloud_resource_id_info_map[cloud_resource_id]) != 1):
meter_id = flavor_params['meter_id']
current_r_info = [
x for x in cloud_resource_id_info_map[
cloud_resource_id] if x['meter_id'] == meter_id][0]

instance = cloud_resource_id_instance_map[cloud_resource_id]
metric = metrics_map.get(instance['_id'])
instance_key = self.get_record_key(instance)
optimization = optimizations_dict[instance_key]
if metric is None:
self._set_reason_properties(
optimization, ArchiveReason.RECOMMENDATION_APPLIED)
optimization, ArchiveReason.FAILED_DEPENDENCY,
'metric not found')
result.append(optimization)
continue
else:
cloud_resource_map[instance['_id']] = instance
cloud_resource_id_instance_map[instance['cloud_resource_id']] = instance

if not cloud_resource_id_instance_map:
continue

cloud_resource_id_info_map = self._get_instances_info(
list(cloud_resource_map.values()), cloud_account_id,
cloud_account['type'])
metrics_map = self._get_metrics(
list(cloud_resource_map.keys()), cloud_account_id,
days_threshold, cloud_account['type'])
current_flavor_params = self._get_flavor_params(
cloud_resource_id_info_map, cloud_resource_id_instance_map,
cloud_account['type'])

for params in current_flavor_params:
cloud_resource_ids = params['resource_ids']
region = params['region']
family_specs = params['family_specs']
flavor_params = params['flavor_params']
if cloud_account['type'] == NEBIUS_CLOUD_TYPE:
flavor_params['cloud_account_id'] = cloud_account_id
current_flavor = self._find_flavor(
cloud_account['type'], region, family_specs, 'current',
**flavor_params)
if not current_flavor:
for cloud_resource_id in cloud_resource_ids:
instance = cloud_resource_id_instance_map[cloud_resource_id]
instance_key = self.get_record_key(instance)
optimization = optimizations_dict[instance_key]
self._set_reason_properties(
optimization, ArchiveReason.FAILED_DEPENDENCY,
'flavor not found')
result.append(optimization)
continue

cpu_flavor_map = {}
for cloud_resource_id in cloud_resource_ids:
current_r_info = cloud_resource_id_info_map[
cloud_resource_id][0]
if (cloud_account['type'] == 'azure_cnr' and len(
cloud_resource_id_info_map[cloud_resource_id]) != 1):
meter_id = flavor_params['meter_id']
current_r_info = [
x for x in cloud_resource_id_info_map[
cloud_resource_id] if x['meter_id'] == meter_id][0]

instance = cloud_resource_id_instance_map[cloud_resource_id]
metric = metrics_map.get(instance['_id'])
instance_key = self.get_record_key(instance)
optimization = optimizations_dict[instance_key]
if metric is None:
recommended_cpu = optimization['recommended_flavor_cpu']
if not cpu_flavor_map.get(recommended_cpu):
flavor_params['cpu'] = recommended_cpu
recommended_flavor = self._find_flavor(
cloud_account['type'], region, family_specs,
'search_relevant', **flavor_params)
if not recommended_flavor:
self._set_reason_properties(
optimization, ArchiveReason.FAILED_DEPENDENCY,
'metric not found')
'recommended flavor not found')
result.append(optimization)
continue
recommended_cpu = optimization['recommended_flavor_cpu']
if not cpu_flavor_map.get(recommended_cpu):
flavor_params['cpu'] = recommended_cpu
recommended_flavor = self._find_flavor(
cloud_account['type'], region, family_specs,
'search_relevant', **flavor_params)
if not recommended_flavor:
self._set_reason_properties(
optimization, ArchiveReason.FAILED_DEPENDENCY,
'recommended flavor not found')
result.append(optimization)
continue
cpu_flavor_map[recommended_cpu] = recommended_flavor
else:
recommended_flavor = cpu_flavor_map[recommended_cpu]
current_cost = current_r_info.get('day_cost', 0) * DAYS_IN_MONTH
if not current_cost:
continue
discount_multiplier = current_r_info.get(
'discount_multiplier', 1)
recommended_cost = (
recommended_flavor.get('price') * HOURS_IN_DAY *
DAYS_IN_MONTH * discount_multiplier)
if recommended_cost >= current_cost:
self._set_reason_properties(
optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT)
else:
self._set_reason_properties(
optimization, ArchiveReason.OPTIONS_CHANGED)
result.append(optimization)
return result
cpu_flavor_map[recommended_cpu] = recommended_flavor
else:
recommended_flavor = cpu_flavor_map[recommended_cpu]
current_cost = current_r_info.get('day_cost',
0) * DAYS_IN_MONTH
if not current_cost:
continue
discount_multiplier = current_r_info.get(
'discount_multiplier', 1)
recommended_cost = (
recommended_flavor.get('price') * HOURS_IN_DAY *
DAYS_IN_MONTH * discount_multiplier)
if recommended_cost >= current_cost:
self._set_reason_properties(
optimization, ArchiveReason.RECOMMENDATION_IRRELEVANT)
else:
self._set_reason_properties(
optimization, ArchiveReason.OPTIONS_CHANGED)
result.append(optimization)


def main(organization_id, config_client, created_at, **kwargs):
Expand Down
27 changes: 27 additions & 0 deletions bumiworker/bumiworker/modules/archive/rightsizing_rds.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
from bumiworker.bumiworker.consts import ArchiveReason
from bumiworker.bumiworker.modules.recommendations.rightsizing_rds import (
RightsizingRds as RightsizingRdsRecommendation)
from bumiworker.bumiworker.modules.rightsizing_base import (
RightsizingArchiveBase
)


class RightsizingRds(RightsizingArchiveBase, RightsizingRdsRecommendation):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.reason_description_map[ArchiveReason.RECOMMENDATION_APPLIED] = (
'flavor changed')

def set_additional_reasons(self, cloud_resource_map, cloud_account,
cloud_resource_id_instance_map,
optimizations_dict, days_threshold, result):
for optimization in optimizations_dict.values():
if 'reason' not in optimization:
self._set_reason_properties(
optimization, ArchiveReason.OPTIONS_CHANGED)
result.append(optimization)


def main(organization_id, config_client, created_at, **kwargs):
return RightsizingRds(
organization_id, config_client, created_at).get()
Loading

0 comments on commit 4faea0a

Please sign in to comment.