Skip to content

Commit

Permalink
OS-4890. instance_subscription archive recommendation
Browse files Browse the repository at this point in the history
  • Loading branch information
nk-hystax authored Jun 24, 2024
1 parent 3af8f3e commit abf236b
Show file tree
Hide file tree
Showing 2 changed files with 154 additions and 41 deletions.
93 changes: 93 additions & 0 deletions bumiworker/bumiworker/modules/archive/instance_subscription.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import logging

from collections import defaultdict
from datetime import datetime, timedelta, timezone

from bumiworker.bumiworker.consts import ArchiveReason
from bumiworker.bumiworker.modules.base import ArchiveBase
from bumiworker.bumiworker.modules.recommendations.instance_subscription import (
InstanceSubscription as InstanceSubscriptionRecommendation,
SUPPORTED_CLOUD_TYPES, SUBSCRIPTION_ITEM
)

LOG = logging.getLogger(__name__)


class InstanceSubscription(ArchiveBase, InstanceSubscriptionRecommendation):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.reason_description_map[ArchiveReason.RESOURCE_DELETED] = (
'instance deleted')
self.reason_description_map[ArchiveReason.FAILED_DEPENDENCY] = (
'subscription unavailable')
self.reason_description_map[ArchiveReason.RECOMMENDATION_IRRELEVANT] = (
'discounts applied')

@property
def supported_cloud_types(self):
return SUPPORTED_CLOUD_TYPES

def _has_discounts(self, raw_info):
if raw_info.get('cost') == 0:
# savings plan applied
return True
for key in ['coupons_discount', 'resource_package_discount']:
if key in raw_info and float(raw_info[key]):
return True

def _get(self, previous_options, optimizations, cloud_accounts_map,
**kwargs):
now = datetime.now(tz=timezone.utc)
days_threshold = previous_options['days_threshold']
range_start_ts = int(
(now - timedelta(days=days_threshold)).timestamp())

account_optimizations_map = defaultdict(list)
for optimization in optimizations:
account_optimizations_map[optimization['cloud_account_id']].append(
optimization)

cloud_acc_instance_map = self.get_cloud_acc_instances_map(
list(account_optimizations_map.keys()), range_start_ts
)

result = []
for cloud_account_id, optimizations_ in account_optimizations_map.items():
if cloud_account_id not in cloud_accounts_map:
for optimization in optimizations_:
self._set_reason_properties(
optimization, ArchiveReason.CLOUD_ACCOUNT_DELETED)
result.append(optimization)
continue

cloud_resource_ids = [x['cloud_resource_id']
for x in optimizations_]
raw_expenses = self.get_raw_expenses(
cloud_account_id, now, cloud_resource_ids)
raw_expenses_map = {x['_id']: x for x in raw_expenses}
active_resources = cloud_acc_instance_map.get(cloud_account_id, {})

for optimization in optimizations_:
cloud_resource_id = optimization['cloud_resource_id']
instance = active_resources.get(cloud_resource_id)
raw_info = raw_expenses_map.get(cloud_resource_id, {})
if not instance:
reason = ArchiveReason.RESOURCE_DELETED
elif SUBSCRIPTION_ITEM in raw_info['billing_item']:
reason = ArchiveReason.RECOMMENDATION_APPLIED
elif self._has_discounts(raw_info):
reason = ArchiveReason.RECOMMENDATION_IRRELEVANT
elif any(x is None for x in self.get_subscriptions_costs(
cloud_resource_id, instance['meta']['flavor'],
instance['region'])):
reason = ArchiveReason.FAILED_DEPENDENCY
else:
reason = ArchiveReason.OPTIONS_CHANGED
self._set_reason_properties(optimization, reason)
result.append(optimization)
return result


def main(organization_id, config_client, created_at, **kwargs):
return InstanceSubscription(
organization_id, config_client, created_at).get()
102 changes: 61 additions & 41 deletions bumiworker/bumiworker/modules/recommendations/instance_subscription.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,17 @@ def insider_cl(self):
verify=False)
return self._insider_cl

def handle_account(self, cloud_account, instance_map, now, excluded_pools):
raw_expenses = self.mongo_client.restapi.raw_expenses.aggregate([
def get_raw_expenses(self, cloud_account_id, now, cloud_resource_ids):
return self.mongo_client.restapi.raw_expenses.aggregate([
{
'$match': {
'$and': [
{'resource_id': {'$in': list(instance_map.keys())}},
{'start_date': {'$gte': now - timedelta(days=DAYS_IN_MONTH)}},
{'cloud_account_id': cloud_account['id']},
{'BillingItem': {'$in': [PAY_AS_YOU_GO_ITEM, SUBSCRIPTION_ITEM]}}
{'resource_id': {'$in': cloud_resource_ids}},
{'start_date': {
'$gte': now - timedelta(days=DAYS_IN_MONTH)}},
{'cloud_account_id': cloud_account_id},
{'BillingItem': {
'$in': [PAY_AS_YOU_GO_ITEM, SUBSCRIPTION_ITEM]}}
]
}
},
Expand All @@ -60,56 +62,68 @@ def handle_account(self, cloud_account, instance_map, now, excluded_pools):
'_id': '$resource_id',
"cost": {'$sum': '$cost'},
'billing_item': {'$addToSet': '$BillingItem'},
'daily_cost_without_discount': {'$max': '$PretaxGrossAmount'},
'daily_cost_without_discount': {
'$max': '$PretaxGrossAmount'
},
'invoice_discount': {'$max': '$InvoiceDiscount'},
'coupons_discount': {'$last': '$DeductedByCoupons'},
'resource_package_discount': {'$last': '$DeductedByResourcePackage'}
'resource_package_discount': {
'$last': '$DeductedByResourcePackage'
}
},
}
])

def get_subscriptions_costs(self, cloud_resource_id, flavor, region):
try:
_, monthly_flavor_prices = self.insider_cl.get_flavor_prices(
ALIBABA_CLOUD_TYPE, flavor, region, os_type='linux',
billing_method='subscription', quantity=1)
_, yearly_flavor_prices = self.insider_cl.get_flavor_prices(
ALIBABA_CLOUD_TYPE, flavor, region, os_type='linux',
billing_method='subscription', quantity=MONTHS_IN_YEAR)
monthly_subscription_cost = monthly_flavor_prices['prices'][0].get(
'price')
yearly_subscription_cost = yearly_flavor_prices['prices'][0].get(
'price')
except (HTTPError, KeyError) as ex:
LOG.info('Instance %s skipped due to inability to get '
'subscription price - %s', cloud_resource_id, str(ex))
return None, None
return monthly_subscription_cost, yearly_subscription_cost

def handle_account(self, cloud_account, instance_map, now, excluded_pools):
raw_expenses = self.get_raw_expenses(cloud_account['id'], now,
list(instance_map.keys()))
result = []
for raw_info in raw_expenses:
cloud_resource_id = raw_info['_id']
if SUBSCRIPTION_ITEM in raw_info['billing_item']:
LOG.warning('Instance %s skipped due to detected '
'subscription costs', raw_info['_id'])
continue
if raw_info['coupons_discount'] and float(
raw_info['coupons_discount']):
LOG.info('Instance %s skipped due to coupons discounts',
raw_info['_id'])
cloud_resource_id)
continue
if raw_info['resource_package_discount'] and float(
raw_info['resource_package_discount']):
LOG.info('Instance %s skipped due to package discounts',
raw_info['_id'])
cloud_resource_id)
continue
if raw_info['cost'] == 0:
LOG.info('Instance %s skipped due to saving plan',
raw_info['_id'])
cloud_resource_id)
continue

instance = instance_map.get(raw_info['_id'])
instance = instance_map.get(cloud_resource_id)
flavor = instance['meta']['flavor']
region = instance['region']
try:
_, monthly_flavor_prices = self.insider_cl.get_flavor_prices(
ALIBABA_CLOUD_TYPE, flavor, region, os_type='linux',
billing_method='subscription', quantity=1)
_, yearly_flavor_prices = self.insider_cl.get_flavor_prices(
ALIBABA_CLOUD_TYPE, flavor, region, os_type='linux',
billing_method='subscription', quantity=MONTHS_IN_YEAR)
except HTTPError as ex:
LOG.info('Instance %s skipped due to inability to get '
'subscription price - %s', raw_info['_id'], str(ex))
continue
# both prices are discounted
monthly_subscription_cost = monthly_flavor_prices['prices'][0].get(
'price')
yearly_subscription_cost = yearly_flavor_prices['prices'][0].get(
'price')
(monthly_subscription_cost,
yearly_subscription_cost) = self.get_subscriptions_costs(
cloud_resource_id, flavor, region)
if not monthly_subscription_cost or not yearly_subscription_cost:
LOG.info('Instance %s skipped due to inability to get '
'subscription price', raw_info['_id'])
continue
raw_cost = float(raw_info['daily_cost_without_discount'])
discount = float(raw_info['invoice_discount']) / raw_cost
Expand All @@ -121,7 +135,7 @@ def handle_account(self, cloud_account, instance_map, now, excluded_pools):
discounted_monthly_cost - yearly_subscription_cost_per_month)
if saving_per_month <= 0 or yearly_saving_per_month <= 0:
LOG.warning('Instance %s skipped due non-positive savings',
raw_info['_id'])
cloud_resource_id)
continue
result.append({
'monthly_saving': saving_per_month,
Expand All @@ -140,16 +154,7 @@ def handle_account(self, cloud_account, instance_map, now, excluded_pools):
})
return result

def _get(self):
(days_threshold, excluded_pools,
skip_cloud_accounts) = self.get_options_values()

cloud_account_map = self.get_cloud_accounts(
SUPPORTED_CLOUD_TYPES, skip_cloud_accounts)
cloud_account_ids = list(cloud_account_map.keys())
now = datetime.utcnow()
range_start_ts = int((now - timedelta(days=days_threshold)).timestamp())

def get_cloud_acc_instances_map(self, cloud_account_ids, range_start_ts):
instances = list(self.mongo_client.restapi.resources.find({
'$and': [
{'resource_type': 'Instance'},
Expand All @@ -166,6 +171,21 @@ def _get(self):
cloud_acc_instance_map[
instance['cloud_account_id']][
instance['cloud_resource_id']] = instance
return cloud_acc_instance_map

def _get(self):
(days_threshold, excluded_pools,
skip_cloud_accounts) = self.get_options_values()

cloud_account_map = self.get_cloud_accounts(
SUPPORTED_CLOUD_TYPES, skip_cloud_accounts)
cloud_account_ids = list(cloud_account_map.keys())
now = datetime.utcnow()
range_start_ts = int(
(now - timedelta(days=days_threshold)).timestamp())
cloud_acc_instance_map = self.get_cloud_acc_instances_map(
cloud_account_ids, range_start_ts
)

result = []
futures = []
Expand Down

0 comments on commit abf236b

Please sign in to comment.