From 55341f201113ae6af2024f301993522208e90aad Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Tue, 16 Jan 2024 22:46:34 +0100 Subject: [PATCH] Add overprice scanning --- requirements.pip | 3 +- stats-backend/api2/migrations/0006_glm.py | 20 +++ .../0007_offer_monthly_price_usd.py | 18 +++ .../api2/migrations/0008_ec2instance.py | 23 +++ ...overpriced_offer_overpriced_compared_to.py | 24 ++++ .../0010_offer_suggest_env_per_hour_price.py | 18 +++ .../0011_offer_times_more_expensive.py | 18 +++ stats-backend/api2/models.py | 21 +++ stats-backend/api2/serializers.py | 25 ++-- stats-backend/api2/tasks.py | 134 +++++++++++++++++- stats-backend/api2/utils.py | 81 ++++++++++- stats-backend/core/celery.py | 14 ++ 12 files changed, 381 insertions(+), 18 deletions(-) create mode 100644 stats-backend/api2/migrations/0006_glm.py create mode 100644 stats-backend/api2/migrations/0007_offer_monthly_price_usd.py create mode 100644 stats-backend/api2/migrations/0008_ec2instance.py create mode 100644 stats-backend/api2/migrations/0009_offer_is_overpriced_offer_overpriced_compared_to.py create mode 100644 stats-backend/api2/migrations/0010_offer_suggest_env_per_hour_price.py create mode 100644 stats-backend/api2/migrations/0011_offer_times_more_expensive.py diff --git a/requirements.pip b/requirements.pip index 1e5fd5b..87e7ef3 100644 --- a/requirements.pip +++ b/requirements.pip @@ -85,4 +85,5 @@ web3 eth-account eth-tester PyJWT -djangorestframework-simplejwt \ No newline at end of file +djangorestframework-simplejwt +hcloud \ No newline at end of file diff --git a/stats-backend/api2/migrations/0006_glm.py b/stats-backend/api2/migrations/0006_glm.py new file mode 100644 index 0000000..cfffa8a --- /dev/null +++ b/stats-backend/api2/migrations/0006_glm.py @@ -0,0 +1,20 @@ +# Generated by Django 4.1.7 on 2024-01-16 19:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0005_healtchecktask'), + ] + + operations = [ + migrations.CreateModel( + name='GLM', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('current_price', models.FloatField(null=True)), + ], + ), + ] diff --git a/stats-backend/api2/migrations/0007_offer_monthly_price_usd.py b/stats-backend/api2/migrations/0007_offer_monthly_price_usd.py new file mode 100644 index 0000000..a022354 --- /dev/null +++ b/stats-backend/api2/migrations/0007_offer_monthly_price_usd.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.7 on 2024-01-16 19:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0006_glm'), + ] + + operations = [ + migrations.AddField( + model_name='offer', + name='monthly_price_usd', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/stats-backend/api2/migrations/0008_ec2instance.py b/stats-backend/api2/migrations/0008_ec2instance.py new file mode 100644 index 0000000..6833944 --- /dev/null +++ b/stats-backend/api2/migrations/0008_ec2instance.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.7 on 2024-01-16 21:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0007_offer_monthly_price_usd'), + ] + + operations = [ + migrations.CreateModel( + name='EC2Instance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ('vcpu', models.IntegerField(null=True)), + ('memory', models.FloatField(null=True)), + ('price_usd', models.DecimalField(decimal_places=2, max_digits=10, null=True)), + ], + ), + ] diff --git a/stats-backend/api2/migrations/0009_offer_is_overpriced_offer_overpriced_compared_to.py b/stats-backend/api2/migrations/0009_offer_is_overpriced_offer_overpriced_compared_to.py new file mode 100644 index 0000000..1f69fb1 --- /dev/null +++ b/stats-backend/api2/migrations/0009_offer_is_overpriced_offer_overpriced_compared_to.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.7 on 2024-01-16 21:23 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0008_ec2instance'), + ] + + operations = [ + migrations.AddField( + model_name='offer', + name='is_overpriced', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='offer', + name='overpriced_compared_to', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='api2.ec2instance'), + ), + ] diff --git a/stats-backend/api2/migrations/0010_offer_suggest_env_per_hour_price.py b/stats-backend/api2/migrations/0010_offer_suggest_env_per_hour_price.py new file mode 100644 index 0000000..caa2593 --- /dev/null +++ b/stats-backend/api2/migrations/0010_offer_suggest_env_per_hour_price.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.7 on 2024-01-16 21:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0009_offer_is_overpriced_offer_overpriced_compared_to'), + ] + + operations = [ + migrations.AddField( + model_name='offer', + name='suggest_env_per_hour_price', + field=models.FloatField(null=True), + ), + ] diff --git a/stats-backend/api2/migrations/0011_offer_times_more_expensive.py b/stats-backend/api2/migrations/0011_offer_times_more_expensive.py new file mode 100644 index 0000000..add3b0d --- /dev/null +++ b/stats-backend/api2/migrations/0011_offer_times_more_expensive.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.7 on 2024-01-16 22:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api2', '0010_offer_suggest_env_per_hour_price'), + ] + + operations = [ + migrations.AddField( + model_name='offer', + name='times_more_expensive', + field=models.FloatField(null=True), + ), + ] diff --git a/stats-backend/api2/models.py b/stats-backend/api2/models.py index 389dddb..5e720eb 100644 --- a/stats-backend/api2/models.py +++ b/stats-backend/api2/models.py @@ -17,6 +17,15 @@ class Node(models.Model): created_at = models.DateTimeField(auto_now_add=True) +class EC2Instance(models.Model): + name = models.CharField(max_length=100, unique=True) + vcpu = models.IntegerField(null=True) + memory = models.FloatField(null=True) # Assuming memory is in GB + price_usd = models.DecimalField(max_digits=10, decimal_places=2, null=True) + + def __str__(self): + return self.name + class Offer(models.Model): properties = models.JSONField(null=True) runtime = models.CharField(max_length=42) @@ -24,6 +33,11 @@ class Offer(models.Model): updated_at = models.DateTimeField(auto_now=True) created_at = models.DateTimeField(auto_now_add=True) monthly_price_glm = models.FloatField(null=True, blank=True) + monthly_price_usd = models.FloatField(null=True, blank=True) + is_overpriced = models.BooleanField(default=False) + overpriced_compared_to = models.ForeignKey(EC2Instance, on_delete=models.CASCADE, null=True) + suggest_env_per_hour_price = models.FloatField(null=True) + times_more_expensive = models.FloatField(null=True) class Meta: unique_together = ( @@ -38,3 +52,10 @@ class HealtcheckTask(models.Model): status = models.TextField() updated_at = models.DateTimeField(auto_now=True) created_at = models.DateTimeField(auto_now_add=True) + + +class GLM(models.Model): + current_price = models.FloatField(null=True) + + + diff --git a/stats-backend/api2/serializers.py b/stats-backend/api2/serializers.py index 72b8028..c72e33e 100644 --- a/stats-backend/api2/serializers.py +++ b/stats-backend/api2/serializers.py @@ -1,12 +1,22 @@ from rest_framework import serializers -from .models import Node, Offer +from .models import Node, Offer, EC2Instance +class EC2InstanceSerializer(serializers.ModelSerializer): + class Meta: + model = EC2Instance + fields = '__all__' class OfferSerializer(serializers.ModelSerializer): + overpriced_compared_to = EC2InstanceSerializer(read_only=True) + + class Meta: model = Offer - fields = ["runtime", "monthly_price_glm", "properties", "updated_at"] - + fields = [ + "runtime", "monthly_price_glm", "properties", + "updated_at", "monthly_price_usd", "is_overpriced", + "overpriced_compared_to", "suggest_env_per_hour_price" + ] class NodeSerializer(serializers.ModelSerializer): runtimes = serializers.SerializerMethodField("get_offers") @@ -27,11 +37,4 @@ class Meta: def get_offers(self, node): offers = Offer.objects.filter(provider=node) - data = {} - for obj in offers: - data[obj.runtime] = { - "monthly_price_glm": obj.monthly_price_glm, - "updated_at": obj.updated_at, - "properties": obj.properties, - } - return data + return {offer.runtime: OfferSerializer(offer).data for offer in offers} diff --git a/stats-backend/api2/tasks.py b/stats-backend/api2/tasks.py index 7968103..c6651f5 100644 --- a/stats-backend/api2/tasks.py +++ b/stats-backend/api2/tasks.py @@ -3,7 +3,7 @@ import json import subprocess import os -from .models import Node, Offer +from .models import Node, Offer, GLM, EC2Instance from django.utils import timezone import tempfile import redis @@ -13,7 +13,10 @@ import requests from api.serializers import FlatNodeSerializer from collector.models import Node as NodeV1 - +from django.db.models import F +from django.db.models.functions import Abs +from decimal import Decimal +from .utils import get_pricing, get_ec2_products, find_cheapest_price, has_vcpu_memory, round_to_three_decimals pool = redis.ConnectionPool(host="redis", port=6379, db=0) r = redis.Redis(connection_pool=pool) @@ -323,6 +326,20 @@ def v2_cheapest_provider(): r.set("v2_cheapest_provider", data) +@app.task +def get_current_glm_price(): + url = "https://api.coingecko.com/api/v3/coins/ethereum/contract/0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429" + response = requests.get(url) + if response.status_code == 200: + data = response.json() + price = str(data['market_data']['current_price']['usd'])[0:5] + obj, created = GLM.objects.get_or_create(id=1) + obj.current_price = price + obj.save() + else: + print("Failed to retrieve data") + + @app.task def v2_offer_scraper(): os.chdir("/stats-backend/yapapi/examples/low-level-api/v2") @@ -336,6 +353,7 @@ def v2_offer_scraper(): now = datetime.datetime.now() days_in_current_month = calendar.monthrange(now.year, now.month)[1] seconds_current_month = days_in_current_month * 24 * 60 * 60 + glm_usd_value = GLM.objects.get(id=1) for line in serialized: data = json.loads(line) provider = data["id"] @@ -361,11 +379,43 @@ def v2_offer_scraper(): vectors["golem.usage.cpu_sec"] ] * seconds_current_month - * data["golem.inf.cpu.cores"] + * data["golem.inf.cpu.threads"] ) + data["golem.com.pricing.model.linear.coeffs"][-1] ) + if not monthly_pricing: + print(f"Monthly price is {monthly_pricing}") offerobj.monthly_price_glm = monthly_pricing + offerobj.monthly_price_usd = monthly_pricing * glm_usd_value.current_price + vcpu_needed = data.get("golem.inf.cpu.threads", 0) + memory_needed = data.get("golem.inf.mem.gib", 0.0) + closest_ec2 = EC2Instance.objects.annotate( + cpu_diff=Abs(F('vcpu') - vcpu_needed), + memory_diff=Abs(F('memory') - memory_needed) + ).order_by('cpu_diff', 'memory_diff', 'price_usd').first() + + # Compare and update the Offer object + if closest_ec2 and monthly_pricing: + offer_is_more_expensive = offerobj.monthly_price_usd > closest_ec2.price_usd + comparison_result = "more expensive" if offer_is_more_expensive else "cheaper" + + # Update Offer object fields + offerobj.is_overpriced = offer_is_more_expensive + if offer_is_more_expensive: + offerobj.overpriced_compared_to = closest_ec2 + offerobj.suggest_env_per_hour_price = round_to_three_decimals(( + closest_ec2.price_usd / + Decimal(glm_usd_value.current_price) / + (seconds_current_month / Decimal(3600)) + )) + offerobj.times_more_expensive = offerobj.monthly_price_usd / closest_ec2.price_usd + else: + offerobj.overpriced_compared_to = None + + else: + print("No matching EC2Instance found or monthly pricing is not available.") + offerobj.is_overpriced = False + offerobj.overpriced_compared_to = None offerobj.save() obj.wallet = wallet # Verify each node's status @@ -395,12 +445,46 @@ def v2_offer_scraper(): vectors["golem.usage.cpu_sec"] ] * seconds_current_month - * data["golem.inf.cpu.cores"] + * data["golem.inf.cpu.threads"] ) + data["golem.com.pricing.model.linear.coeffs"][-1] ) + if not monthly_pricing: + print(f"Monthly price is {monthly_pricing}") offerobj.monthly_price_glm = monthly_pricing - offerobj.save() + offerobj.monthly_price_usd = monthly_pricing * glm_usd_value.current_price + + + vcpu_needed = data.get("golem.inf.cpu.threads", 0) + memory_needed = data.get("golem.inf.mem.gib", 0.0) + closest_ec2 = EC2Instance.objects.annotate( + cpu_diff=Abs(F('vcpu') - vcpu_needed), + memory_diff=Abs(F('memory') - memory_needed) + ).order_by('cpu_diff', 'memory_diff', 'price_usd').first() + + # Compare and update the Offer object + if closest_ec2 and monthly_pricing: + offer_is_more_expensive = offerobj.monthly_price_usd > closest_ec2.price_usd + comparison_result = "more expensive" if offer_is_more_expensive else "cheaper" + + # Update Offer object fields + offerobj.is_overpriced = offer_is_more_expensive + if offer_is_more_expensive: + offerobj.overpriced_compared_to = closest_ec2 + offerobj.suggest_env_per_hour_price = round_to_three_decimals(( + closest_ec2.price_usd / + Decimal(glm_usd_value.current_price) / + (seconds_current_month / Decimal(3600)) + )) + offerobj.times_more_expensive = offerobj.monthly_price_usd / closest_ec2.price_usd + else: + offerobj.overpriced_compared_to = None + + else: + print("No matching EC2Instance found or monthly pricing is not available.") + offerobj.is_overpriced = False + offerobj.overpriced_compared_to = None + offerobj.properties = data offerobj.save() obj.runtime = data["golem.runtime.name"] @@ -450,3 +534,43 @@ def healthcheck_provider(node_id, network, taskId): rc = proc.poll() return rc + + + + + +@app.task +def store_ec2_info(): + ec2_info = {} + products_data = get_ec2_products() + + for product in products_data: + details = product.get('details', {}) + if not has_vcpu_memory(details): + continue + print(product) + product_id = product['id'] + category = product.get('category') + name = product.get('name') + + pricing_data = get_pricing(product_id) + cheapest_price = find_cheapest_price(pricing_data['prices']) + + # Convert memory to float and price to Decimal + memory_gb = float(details['memory']) + price = cheapest_price['amount'] if cheapest_price else None + + # Use get_or_create to store or update the instance in the database + instance, created = EC2Instance.objects.get_or_create( + name=name, + defaults={'vcpu': details['vcpu'], 'memory': memory_gb, 'price_usd': price} + ) + + ec2_info[product_id] = { + 'category': category, + 'name': name, + 'details': details, + 'cheapest_price': cheapest_price + } + + return ec2_info diff --git a/stats-backend/api2/utils.py b/stats-backend/api2/utils.py index 1c22adf..847a64b 100644 --- a/stats-backend/api2/utils.py +++ b/stats-backend/api2/utils.py @@ -1,6 +1,6 @@ import subprocess from django.conf import settings - +import os from .models import Offer def is_provider_online(provider): command = f"yagna net find {provider}" @@ -24,3 +24,82 @@ def identify_network(provider): return "mainnet" return "testnet" + + + + + +import requests +import time + +def make_request_with_rate_limit_handling(url, headers): + while True: + response = requests.get(url, headers=headers) + if response.status_code == 429: # Rate limit hit + + reset_time = int(response.headers.get('x-rate-limit-reset', 0)) + sleep_duration = max(reset_time - time.time(), 0) + print(f"Ratelimited waiting for {sleep_duration}") + time.sleep(sleep_duration + 1) # Sleep until the limit resets, then retry + else: + return response + +def get_ec2_products(): + products = [] + url = 'https://api.vantage.sh/v2/products?service_id=aws-ec2' + headers = { + 'accept': 'application/json', + 'authorization': f'Bearer {os.environ.get("VANTAGE_API_KEY")}' + } + + while url: + response = make_request_with_rate_limit_handling(url, headers) + data = response.json() + print("Got product list") + products.extend(data.get('products', [])) + url = data['links'].get('next') # Get the next page URL + + return products + +def get_pricing(product_id): + url = f'https://api.vantage.sh/v2/products/{product_id}/prices' + headers = { + 'accept': 'application/json', + 'authorization': f'Bearer {os.environ.get("VANTAGE_API_KEY")}' + } + response = make_request_with_rate_limit_handling(url, headers) + print("Got price") + return response.json() + +def find_cheapest_price(prices): + return min(prices, key=lambda x: x['amount']) if prices else None + +def has_vcpu_memory(details): + return 'vcpu' in details and 'memory' in details + +from decimal import Decimal,ROUND_DOWN +def round_to_three_decimals(value): + # Convert to Decimal + decimal_value = Decimal(value) + + # If the value is less than 1 and not zero, handle the first non-zero digit + if 0 < decimal_value < 1: + # Convert to scientific notation to find the first non-zero digit + value_scientific = format(decimal_value, '.6e') + exponent = int(value_scientific.split('e')[-1]) + + # If the exponent is significantly small, handle as a special case + if exponent <= -6: + rounded_value = decimal_value + else: + # Calculate the number of decimal places to keep + decimal_places = abs(exponent) + 2 # 2 more than the exponent + quantize_pattern = Decimal('1e-' + str(decimal_places)) + + # Rounding the value + rounded_value = decimal_value.quantize(quantize_pattern, rounding=ROUND_DOWN) + else: + # If the value is 1 or more, or exactly 0, round to a maximum of three decimal places + rounded_value = decimal_value.quantize(Decimal("0.001"), rounding=ROUND_DOWN) + + return rounded_value \ No newline at end of file diff --git a/stats-backend/core/celery.py b/stats-backend/core/celery.py index b614da0..84a3de5 100644 --- a/stats-backend/core/celery.py +++ b/stats-backend/core/celery.py @@ -50,6 +50,8 @@ def setup_periodic_tasks(sender, **kwargs): latest_blog_posts, v2_cheapest_offer, v2_network_online_to_redis_flatmap, + get_current_glm_price, + store_ec2_info ) sender.add_periodic_task( @@ -58,6 +60,12 @@ def setup_periodic_tasks(sender, **kwargs): queue="yagna", options={"queue": "yagna", "routing_key": "yagna"}, ) + sender.add_periodic_task( + crontab(hour="*/24"), + store_ec2_info.s(), + queue="default", + options={"queue": "default", "routing_key": "default"}, + ) sender.add_periodic_task( crontab(minute="*/60"), fetch_yagna_release.s(), @@ -113,6 +121,12 @@ def setup_periodic_tasks(sender, **kwargs): queue="default", options={"queue": "default", "routing_key": "default"}, ) + sender.add_periodic_task( + 60.0, + get_current_glm_price.s(), + queue="default", + options={"queue": "default", "routing_key": "default"}, + ) # sender.add_periodic_task( # 10.0, # save_endpoint_logs_to_db.s(),