From 4a5ea7e43105a8449ceb0ff0768c9c00eb41ce04 Mon Sep 17 00:00:00 2001 From: Mateo Gonzales Navarrete <38146507+mgonnav@users.noreply.github.com> Date: Wed, 1 Nov 2023 14:48:27 -0500 Subject: [PATCH] [MRG] Add estela proxies (#231) (#234) * [MRG] Add bmc proxies (#231) * Add Estela Proxies. --------- Co-authored-by: mgonnav * Update RESERVED_PROXY_NAMES variable name. Add download size settings as variables * Add PROXY_PROVIDERS_TO_TRACK, MAX_WEB_DOWNLOAD_SIZE_MB, and MAX_CLI_DOWNLOAD_CHUNK_MB to the API docs --------- Co-authored-by: joaquin garmendia --- .gitignore | 4 +- docs/estela/installation/helm-variables.md | 16 + estela-api/api/serializers/job.py | 13 +- estela-api/api/serializers/proxyprovider.py | 23 ++ estela-api/api/urls.py | 5 + estela-api/api/utils.py | 47 ++- estela-api/api/views/job.py | 23 +- estela-api/api/views/job_data.py | 4 +- estela-api/api/views/proxyprovider.py | 85 +++++ estela-api/config/settings/base.py | 23 +- estela-api/core/admin.py | 15 +- .../core/migrations/0034_proxy_provider.py | 80 +++++ estela-api/core/models.py | 48 +++ estela-api/core/tasks.py | 12 + estela-api/docs/api.yaml | 170 ++++++++++ estela-web/.env.development.example | 1 + .../components/EnvVarsSettingsPage/index.tsx | 4 +- .../ProxySettingsPage/ProxyForm.tsx | 152 +++++++++ .../components/ProxySettingsPage/index.tsx | 296 ++++++++++++++++++ .../components/ProxySettingsPage/styles.scss | 33 ++ .../src/components/ProxySettingsPage/types.ts | 25 ++ estela-web/src/constants.ts | 1 + .../src/pages/CronJobDetailPage/index.tsx | 24 +- .../src/pages/CronjobCreateModal/index.tsx | 128 +++++++- estela-web/src/pages/JobCreateModal/index.tsx | 123 +++++++- estela-web/src/pages/JobDataPage/index.tsx | 8 +- estela-web/src/pages/JobDetailPage/index.tsx | 75 +++-- .../src/pages/JobDetailPage/styles.scss | 8 + .../src/pages/ProjectSettingsPage/index.tsx | 28 ++ .../src/pages/SpiderDetailPage/index.tsx | 29 ++ .../generated-api/.openapi-generator/FILES | 4 + .../services/api/generated-api/apis/ApiApi.ts | 247 +++++++++++++++ .../models/InlineResponse2009.ts | 88 ++++++ .../api/generated-api/models/ProxyProvider.ts | 72 +++++ .../models/ProxyProviderResponse.ts | 72 +++++ .../models/ProxyProviderUpdate.ts | 65 ++++ .../api/generated-api/models/index.ts | 4 + estela-web/src/utils.ts | 35 +++ installation/Makefile | 24 +- installation/data.json | 0 .../templates/API/api-configmap.yaml | 2 +- installation/helm-chart/values.yaml.example | 7 + 42 files changed, 2044 insertions(+), 79 deletions(-) create mode 100644 estela-api/api/serializers/proxyprovider.py create mode 100644 estela-api/api/views/proxyprovider.py create mode 100644 estela-api/core/migrations/0034_proxy_provider.py create mode 100644 estela-web/src/components/ProxySettingsPage/ProxyForm.tsx create mode 100644 estela-web/src/components/ProxySettingsPage/index.tsx create mode 100644 estela-web/src/components/ProxySettingsPage/styles.scss create mode 100644 estela-web/src/components/ProxySettingsPage/types.ts create mode 100644 estela-web/src/services/api/generated-api/models/InlineResponse2009.ts create mode 100644 estela-web/src/services/api/generated-api/models/ProxyProvider.ts create mode 100644 estela-web/src/services/api/generated-api/models/ProxyProviderResponse.ts create mode 100644 estela-web/src/services/api/generated-api/models/ProxyProviderUpdate.ts create mode 100644 installation/data.json diff --git a/.gitignore b/.gitignore index a7e04b24..df02fdda 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,6 @@ __pycache__/ .DS_Store # Certificates -*.crt \ No newline at end of file +*.crt + +bitmaker_billing/ \ No newline at end of file diff --git a/docs/estela/installation/helm-variables.md b/docs/estela/installation/helm-variables.md index 207138ed..f6b331c4 100644 --- a/docs/estela/installation/helm-variables.md +++ b/docs/estela/installation/helm-variables.md @@ -173,6 +173,22 @@ All the queue platform variables should be written as children of the _ The mailing configuration is used to send email regarding users creation on the estela system. +#### Data Downloads +* __ (Required): This is the maximum size of the chunks when downloading data + via the [estela-cli](https://estela-cli.bitmaker.la/). E.g., if this is set to a value of 2 and you download 1GB of data, 500 chunks would be + downloaded. +* __ (Required): This is the maximum download size via Estela's web interface. + We recommend not setting this value higher than 2GB, and you should update the timeout value for your API + according to the value you set here. E.g., if you use `gunicorn`, you would add the `timeout` flag: + `gunicorn config.wsgi --bind=0.0.0.0:8000 --timeout=600`. We nencourage you to use the estela-cli for bigger + downloads. + +#### Proxies + +* __ (Optional): In Estela, you can add custom proxy providers you + can configure and reutilize in your projects, spiders, jobs and cronjobs. In this variable, + set the names of the proxy providers you want to track. E.g., `my_custom_proxy,my_other_custom_proxy`. + ### estela queueing variables * __ (Required): Set this value to `"False"` if the database diff --git a/estela-api/api/serializers/job.py b/estela-api/api/serializers/job.py index e7d17929..a973fee2 100644 --- a/estela-api/api/serializers/job.py +++ b/estela-api/api/serializers/job.py @@ -142,7 +142,13 @@ class SpiderJobUpdateSerializer(serializers.ModelSerializer): SpiderJob.RUNNING_STATUS, ] - job_fields = ["lifespan", "total_response_bytes", "item_count", "request_count"] + job_fields = [ + "lifespan", + "total_response_bytes", + "item_count", + "request_count", + "proxy_usage_data", + ] class Meta: model = SpiderJob @@ -155,6 +161,7 @@ class Meta: "request_count", "data_status", "data_expiry_days", + "proxy_usage_data", ) def update(self, instance, validated_data): @@ -188,7 +195,9 @@ def update(self, instance, validated_data): instance.status = status for field in self.job_fields: - if not getattr(instance, field): + if not getattr(instance, field) or getattr( + instance, field + ) != validated_data.get(field): new_value = validated_data.get(field, getattr(instance, field)) setattr(instance, field, new_value) diff --git a/estela-api/api/serializers/proxyprovider.py b/estela-api/api/serializers/proxyprovider.py new file mode 100644 index 00000000..598c7768 --- /dev/null +++ b/estela-api/api/serializers/proxyprovider.py @@ -0,0 +1,23 @@ +from rest_framework import serializers +from core.models import ProxyProvider +from api.serializers.job_specific import SpiderJobEnvVarSerializer + + +class ProxyProviderSerializer(serializers.ModelSerializer): + class Meta: + model = ProxyProvider + fields = ["name", "description", "proxyid"] + + +class ProxyProviderUpdateSerializer(serializers.Serializer): + level = serializers.CharField(max_length=100, help_text="Spider or project") + project_or_spider_id = serializers.CharField( + max_length=100, help_text="Project id where the update will be performed" + ) + + +class ProxyProviderResponseSerializer(serializers.Serializer): + success = serializers.BooleanField() + env_vars = SpiderJobEnvVarSerializer( + many=True, required=False, help_text="Env vars for the instace(project, spider)" + ) diff --git a/estela-api/api/urls.py b/estela-api/api/urls.py index ad4601a2..f2280b51 100644 --- a/estela-api/api/urls.py +++ b/estela-api/api/urls.py @@ -10,6 +10,7 @@ job_data as job_data_views, stats as stats_views, notification as notification_views, + proxyprovider as proxyprovider_views, ) router = routers.DefaultRouter(trailing_slash=False) @@ -58,6 +59,10 @@ viewset=stats_views.SpidersJobsStatsViewSet, basename="spider-stats", ) +router.register( + prefix=r"proxy_provider", + viewset=proxyprovider_views.ProxyProviderViewSet, +) router.register(prefix=r"auth", viewset=auth_views.AuthAPIViewSet, basename="auth") router.register( prefix=r"auth/profile", viewset=auth_views.UserProfileViewSet, basename="profile" diff --git a/estela-api/api/utils.py b/estela-api/api/utils.py index b74a2bb6..f6749a95 100644 --- a/estela-api/api/utils.py +++ b/estela-api/api/utils.py @@ -6,10 +6,10 @@ from api import errors from api.exceptions import DataBaseError from config.job_manager import spiderdata_db_client -from core.models import SpiderJobEnvVar +from core.models import SpiderJobEnvVar, ProxyProvider -def update_env_vars(instance, env_vars, level="project"): +def update_env_vars(instance, env_vars, level="project", delete=True): env_vars_instance = instance.env_vars.all() for env_var in env_vars: if env_vars_instance.filter(**env_var).exists(): @@ -29,9 +29,10 @@ def update_env_vars(instance, env_vars, level="project"): elif level == "spider": SpiderJobEnvVar.objects.create(spider=instance, **env_var) - for env_var in env_vars_instance: - if env_var.name not in [value["name"] for value in env_vars]: - env_var.delete() + if delete: + for env_var in env_vars_instance: + if env_var.name not in [value["name"] for value in env_vars]: + env_var.delete() def update_stats_from_redis(job, save_to_database=False): @@ -72,3 +73,39 @@ def delete_stats_from_redis(job): redis_conn.delete(f"scrapy_stats_{job.key}") except: pass + + +def get_proxy_provider_envs(proxy_id): + proxy_provider = ProxyProvider.objects.get(pk=proxy_id) + proxy_attrs = [ + "username", + "password", + "host", + "port", + "name", + ] + fields_and_values = vars(proxy_provider) + replaces = { + "password": "pass", + "host": "url", + "username": "user", + } + env_vars = [] + for field, value in fields_and_values.items(): + if field in proxy_attrs: + name = replaces.get(field, field).upper() + if name != "NAME": + masked = True + else: + masked = False + env_vars.append( + {"name": f"ESTELA_PROXY_{name}", "value": value, "masked": masked} + ) + env_vars.append( + { + "name": "ESTELA_PROXIES_ENABLED", + "value": "True", + "masked": False, + } + ) + return env_vars diff --git a/estela-api/api/views/job.py b/estela-api/api/views/job.py index 597883bd..8dd9fca5 100644 --- a/estela-api/api/views/job.py +++ b/estela-api/api/views/job.py @@ -13,9 +13,9 @@ SpiderJobSerializer, SpiderJobUpdateSerializer, ) -from api.utils import update_stats_from_redis +from api.utils import update_stats_from_redis, get_proxy_provider_envs from config.job_manager import job_manager -from core.models import DataStatus, Project, Spider, SpiderJob +from core.models import DataStatus, Project, Spider, SpiderJob, ProxyProvider class SpiderJobViewSet( @@ -120,6 +120,25 @@ def create(self, request, *args, **kwargs): job_env_vars = { env_var.name: env_var.value for env_var in job.env_vars.all() } + + proxy_provider_names = [ + (proxy.name, proxy.proxyid) for proxy in ProxyProvider.objects.all() + ] + proxy_name = job_env_vars.get("ESTELA_PROXY_NAME") + + if proxy_name: + proxy_id = next( + (tup[1] for tup in proxy_provider_names if proxy_name in tup), None + ) + if proxy_id: + proxy_env_vars = get_proxy_provider_envs(proxy_id) + job_env_vars.update( + { + env_var["name"]: env_var["value"] + for env_var in proxy_env_vars + } + ) + token = request.auth.key if request.auth else None job_manager.create_job( job.name, diff --git a/estela-api/api/views/job_data.py b/estela-api/api/views/job_data.py index 933652a6..97e8b7f1 100644 --- a/estela-api/api/views/job_data.py +++ b/estela-api/api/views/job_data.py @@ -119,7 +119,7 @@ def list(self, request, *args, **kwargs): elif request.META["HTTP_USER_AGENT"].startswith("estela-cli/"): chunk_size = max( 1, - settings.MAX_CHUNK_SIZE + settings.MAX_CLI_DOWNLOAD_CHUNK_SIZE // spiderdata_db_client.get_estimated_document_size( kwargs["pid"], job_collection_name ), @@ -207,7 +207,7 @@ def download(self, request, *args, **kwargs): else: docs_limit = max( 1, - (settings.MAX_WEB_DOWNLOAD_SIZE) + settings.MAX_WEB_DOWNLOAD_SIZE // spiderdata_db_client.get_estimated_document_size( kwargs["pid"], job_collection_name ), diff --git a/estela-api/api/views/proxyprovider.py b/estela-api/api/views/proxyprovider.py new file mode 100644 index 00000000..6fc887a8 --- /dev/null +++ b/estela-api/api/views/proxyprovider.py @@ -0,0 +1,85 @@ +from rest_framework import viewsets, status +from rest_framework.response import Response +from rest_framework import serializers +from core.models import ProxyProvider, Project, Spider, SpiderJobEnvVar, SpiderJob +from api.serializers.proxyprovider import ( + ProxyProviderUpdateSerializer, + ProxyProviderSerializer, + ProxyProviderResponseSerializer, +) +from api.serializers.job_specific import SpiderJobEnvVarSerializer +from api.mixins import BaseViewSet, ActionHandlerMixin +from drf_yasg.utils import swagger_auto_schema +from api.utils import update_env_vars + +# from utils import update_env_vars + + +class ProxyProviderViewSet(BaseViewSet, viewsets.ModelViewSet, ActionHandlerMixin): + queryset = ProxyProvider.objects.all() + serializer_class = ProxyProviderSerializer + + @swagger_auto_schema( + request_body=ProxyProviderUpdateSerializer, # Especifica el serializer para la solicitud + responses={ + status.HTTP_200_OK: ProxyProviderResponseSerializer() + }, # Define las respuestas + ) + def update(self, request, *args, **kwargs): + "In the request we should specify spider, project or job" + serializer = ProxyProviderUpdateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + if serializer.validated_data["level"] == "project": + instance = Project.objects.get( + pk=serializer.validated_data["project_or_spider_id"] + ) + elif serializer.validated_data["level"] == "spider": + instance = SpiderJob.objects.get( + pk=serializer.validated_data["project_or_spider_id"] + ) + proxy_provider = self.get_object() + proxy_attrs = [ + "username", + "password", + "host", + "port", + "name", + ] + fields_and_values = vars(proxy_provider) + replaces = { + "password": "pass", + "host": "url", + "username": "user", + } + env_vars = [] + for field, value in fields_and_values.items(): + if field in proxy_attrs: + name = replaces.get(field, field).upper() + if name != "NAME": + masked = True + else: + masked = False + env_vars.append( + {"name": f"ESTELA_PROXY_{name}", "value": value, "masked": masked} + ) + update_env_vars( + instance, env_vars, level=serializer.validated_data["level"], delete=False + ) + + if serializer.validated_data["level"] == "project": + env_vars_instance = SpiderJobEnvVar.objects.filter( + project_id=serializer.validated_data["project_or_spider_id"] + ) + if serializer.validated_data["level"] == "spider": + env_vars_instance = SpiderJobEnvVar.objects.filter( + spider_id=serializer.validated_data["project_or_spider_id"] + ) + env_vars_serialized = SpiderJobEnvVarSerializer( + env_vars_instance, required=False, many=True + ) + resp_serializer = ProxyProviderResponseSerializer( + data={"success": True, "env_vars": env_vars_serialized.data} + ) + # response_ser = ProxyProviderResponseSerializer(data=rspse) + resp_serializer.is_valid() + return Response(resp_serializer.data, status=status.HTTP_200_OK) diff --git a/estela-api/config/settings/base.py b/estela-api/config/settings/base.py index 6bf59a36..428d6739 100644 --- a/estela-api/config/settings/base.py +++ b/estela-api/config/settings/base.py @@ -46,6 +46,8 @@ AWS_STORAGE_BUCKET_NAME=(str, "estela-django-api"), GOOGLE_APPLICATION_CREDENTIALS=(str, "dummy"), GOOGLE_APPLICATION_LOCATION=(str, "dummy"), + MAX_CLI_DOWNLOAD_CHUNK_MB=(int, 2), + MAX_WEB_DOWNLOAD_SIZE_MB=(int, 1024), MULTI_NODE_MODE=(str, "False"), BUCKET_NAME_PROJECTS=(str, "dummy"), SECRET_KEY=(str, "dummy"), @@ -63,6 +65,7 @@ EMAIL_HOST=(str, "dummy"), EMAIL_PORT=(int, "dummy"), VERIFICATION_EMAIL=(str, "dummy"), + PROXY_PROVIDERS_TO_TRACK=(str, ""), ) environ.Env.read_env(env_file=".env") @@ -210,8 +213,8 @@ # API limit data download settings (bytes) -MAX_CHUNK_SIZE = 2 * 1024 * 1024 -MAX_WEB_DOWNLOAD_SIZE = 100 * 1024 * 1024 +MAX_CLI_DOWNLOAD_CHUNK_SIZE = env("MAX_CLI_DOWNLOAD_CHUNK_MB") * 1024 * 1024 +MAX_WEB_DOWNLOAD_SIZE = env("MAX_WEB_DOWNLOAD_SIZE_MB") * 1024 * 1024 # Pagination settings used in api_app API_PAGE_SIZE = 100 # Paginator page size @@ -280,7 +283,7 @@ CREDENTIALS = env("CREDENTIALS") SPIDERDATA_DB_ENGINE = env("SPIDERDATA_DB_ENGINE") -# Spiderdata Database settings +# Spiderdata database settings SPIDERDATA_DB_CONNECTION = env("SPIDERDATA_DB_CONNECTION") SPIDERDATA_DB_PRODUCTION = True SPIDERDATA_DB_CERTIFICATE_PATH = env("SPIDERDATA_DB_CERTIFICATE_PATH") @@ -293,8 +296,18 @@ EMAIL_PORT = env("EMAIL_PORT") EMAILS_TO_ALERT = env("EMAILS_TO_ALERT") -# Accept new Users +# Enable/disable the user register endpoint REGISTER = env("REGISTER") -# Verification Email +# Verification email VERIFICATION_EMAIL = env("VERIFICATION_EMAIL") + +# Proxy Settings +PROXY_PROVIDERS_TO_TRACK = ( + [] + if env("PROXY_PROVIDERS_TO_TRACK") == "" + else [ + (name.replace("_", " ").title(), f"{name}_usage") + for name in env("PROXY_PROVIDERS_TO_TRACK").split(",") + ] +) diff --git a/estela-api/core/admin.py b/estela-api/core/admin.py index 3272254b..d79eb07e 100644 --- a/estela-api/core/admin.py +++ b/estela-api/core/admin.py @@ -1,13 +1,14 @@ -from django.contrib import admin - from core.models import ( Project, + ProxyProvider, Spider, SpiderJob, SpiderJobArg, SpiderJobEnvVar, UsageRecord, + Permission, ) +from django.contrib import admin @admin.register(Project) @@ -15,6 +16,11 @@ class ProjectAdmin(admin.ModelAdmin): pass +@admin.register(Permission) +class PermissionAdmin(admin.ModelAdmin): + pass + + @admin.register(Spider) class SpiderAdmin(admin.ModelAdmin): pass @@ -39,3 +45,8 @@ class SpiderJobAdmin(admin.ModelAdmin): @admin.register(UsageRecord) class UsageRecordAdmin(admin.ModelAdmin): pass + + +@admin.register(ProxyProvider) +class ProxyProviderAdmin(admin.ModelAdmin): + pass diff --git a/estela-api/core/migrations/0034_proxy_provider.py b/estela-api/core/migrations/0034_proxy_provider.py new file mode 100644 index 00000000..2281f874 --- /dev/null +++ b/estela-api/core/migrations/0034_proxy_provider.py @@ -0,0 +1,80 @@ +# Generated by Django 3.1.14 on 2023-10-09 17:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0033_auto_20230814_2238"), + ] + + operations = [ + migrations.CreateModel( + name="ProxyProvider", + fields=[ + ( + "proxyid", + models.AutoField( + help_text="A unique integer value identifying this proxy.", + primary_key=True, + serialize=False, + ), + ), + ( + "username", + models.CharField( + help_text="The username for the proxy", max_length=255 + ), + ), + ( + "password", + models.CharField( + help_text="The password for the proxy", max_length=255 + ), + ), + ( + "host", + models.CharField( + help_text="The host for the proxy", max_length=255 + ), + ), + ( + "port", + models.CharField(help_text="The port for the proxy", max_length=5), + ), + ( + "name", + models.CharField( + help_text="A name to identify the proxy", max_length=255 + ), + ), + ( + "description", + models.CharField( + help_text="A description for the proxy", max_length=1000 + ), + ), + ], + ), + migrations.AddField( + model_name="spiderjob", + name="proxy_usage_data", + field=models.JSONField(default=dict, help_text="Proxy Usage data."), + ), + migrations.AddField( + model_name="usagerecord", + name="datacenter_proxy_usage", + field=models.PositiveBigIntegerField( + default=0, + help_text="Amount in bytes occupied by datacenter proxy responses in the database", + ), + ), + migrations.AddField( + model_name="usagerecord", + name="residential_proxy_usage", + field=models.PositiveBigIntegerField( + default=0, + help_text="Amount in bytes occupied by residential proxy responses in the database", + ), + ), + ] diff --git a/estela-api/core/models.py b/estela-api/core/models.py index 239c8bbd..fd5cf672 100644 --- a/estela-api/core/models.py +++ b/estela-api/core/models.py @@ -301,6 +301,15 @@ class SpiderJob(models.Model): request_count = models.PositiveBigIntegerField( default=0, help_text="The number of requests made by the spider job." ) + # proxy_usage_data follows this format: + # { + # "proxy_name": , + # "bytes": , + # } + proxy_usage_data = models.JSONField( + default=dict, + help_text="Proxy Usage data.", + ) class Meta: ordering = ["-created"] @@ -359,6 +368,10 @@ class SpiderJobEnvVar(models.Model): primary_key=True, help_text="A unique integer value identifying this job env variable.", ) + + def __str__(self): + return f"EnvVar ID: {self.evid}, Name: {self.name}, Value: {self.value}, Masked: {self.masked}" + job = models.ForeignKey( SpiderJob, on_delete=models.CASCADE, @@ -431,6 +444,15 @@ class UsageRecord(models.Model): items_data_size = models.PositiveBigIntegerField( help_text="Amount in bytes occupied by items in the database" ) + residential_proxy_usage = models.PositiveBigIntegerField( + default=0, + help_text="Amount in bytes occupied by residential proxy responses in the database", + ) + datacenter_proxy_usage = models.PositiveBigIntegerField( + default=0, + help_text="Amount in bytes occupied by datacenter proxy responses in the database", + ) + requests_data_size = models.PositiveBigIntegerField( help_text="Amount in bytes occupied by requests in the database" ) @@ -487,3 +509,29 @@ class Notification(models.Model): seen = models.BooleanField( default=False, help_text="Whether the notification was seen." ) + + +class ProxyProvider(models.Model): + proxyid = models.AutoField( + primary_key=True, help_text="A unique integer value identifying this proxy." + ) + username = models.CharField(max_length=255, help_text="The username for the proxy") + password = models.CharField(max_length=255, help_text="The password for the proxy") + host = models.CharField(max_length=255, help_text="The host for the proxy") + port = models.CharField(max_length=5, help_text="The port for the proxy") + name = models.CharField(max_length=255, help_text="A name to identify the proxy") + + description = models.CharField( + max_length=1000, help_text="A description for the proxy" + ) + + # You can add a brief help text for the entire model here. + help_text = "A model to store proxy information for your application." + + # Define the on_delete behavior for ForeignKey relationships. + # Since there are no ForeignKey fields in this model, we'll omit this. + + # on_delete=models.CASCADE (example) + + def __str__(self): + return self.name diff --git a/estela-api/core/tasks.py b/estela-api/core/tasks.py index 5e8ca5ff..c68e47ca 100644 --- a/estela-api/core/tasks.py +++ b/estela-api/core/tasks.py @@ -218,6 +218,17 @@ def record_project_usage_after_job_event(job_id): logs_data_size = spiderdata_db_client.get_collection_size( str(project.pid), logs_collection_name ) + # Tracking Proxy Usage + proxy_details = {} + for proxy_name, proxy_usage_name in settings.PROXY_PROVIDERS_TO_TRACK: + proxy_usage_data = json.loads(job.proxy_usage_data) + proxy_details.update( + { + proxy_usage_name: proxy_usage_data["bytes"] + if proxy_name in proxy_usage_data.get("proxy_name") + else 0, + } + ) updated_values = { "processing_time": job.lifespan, @@ -226,6 +237,7 @@ def record_project_usage_after_job_event(job_id): "request_count": job.request_count, "requests_data_size": requests_data_size, "logs_data_size": logs_data_size, + **proxy_details, } last_usage_record = UsageRecord.objects.filter(project=project).first() if last_usage_record: diff --git a/estela-api/docs/api.yaml b/estela-api/docs/api.yaml index dcfaef48..387f5a5b 100644 --- a/estela-api/docs/api.yaml +++ b/estela-api/docs/api.yaml @@ -1302,6 +1302,122 @@ paths: required: true type: string format: uuid + /api/proxy_provider: + get: + operationId: api_proxy_provider_list + description: '' + parameters: + - name: page + in: query + description: A page number within the paginated result set. + required: false + type: integer + - name: page_size + in: query + description: Number of results to return per page. + required: false + type: integer + responses: + '200': + description: '' + schema: + required: + - count + - results + type: object + properties: + count: + type: integer + next: + type: string + format: uri + x-nullable: true + previous: + type: string + format: uri + x-nullable: true + results: + type: array + items: + $ref: '#/definitions/ProxyProvider' + tags: + - api + post: + operationId: api_proxy_provider_create + description: '' + parameters: + - name: data + in: body + required: true + schema: + $ref: '#/definitions/ProxyProvider' + responses: + '201': + description: '' + schema: + $ref: '#/definitions/ProxyProvider' + tags: + - api + parameters: [] + /api/proxy_provider/{proxyid}: + get: + operationId: api_proxy_provider_read + description: '' + parameters: [] + responses: + '200': + description: '' + schema: + $ref: '#/definitions/ProxyProvider' + tags: + - api + put: + operationId: api_proxy_provider_update + description: In the request we should specify spider, project or job + parameters: + - name: data + in: body + required: true + schema: + $ref: '#/definitions/ProxyProviderUpdate' + responses: + '200': + description: '' + schema: + $ref: '#/definitions/ProxyProviderResponse' + tags: + - api + patch: + operationId: api_proxy_provider_partial_update + description: '' + parameters: + - name: data + in: body + required: true + schema: + $ref: '#/definitions/ProxyProvider' + responses: + '200': + description: '' + schema: + $ref: '#/definitions/ProxyProvider' + tags: + - api + delete: + operationId: api_proxy_provider_delete + description: '' + parameters: [] + responses: + '204': + description: '' + tags: + - api + parameters: + - name: proxyid + in: path + description: A unique integer value identifying this proxy. + required: true + type: integer /api/stats/{pid}: get: operationId: api_stats_list @@ -2597,6 +2713,60 @@ definitions: type: integer maximum: 18446744073709551615 minimum: 0 + ProxyProvider: + required: + - name + - description + type: object + properties: + name: + title: Name + description: A name to identify the proxy + type: string + maxLength: 255 + minLength: 1 + description: + title: Description + description: A description for the proxy + type: string + maxLength: 1000 + minLength: 1 + proxyid: + title: Proxyid + description: A unique integer value identifying this proxy. + type: integer + readOnly: true + ProxyProviderUpdate: + required: + - level + - project_or_spider_id + type: object + properties: + level: + title: Level + description: Spider or project + type: string + maxLength: 100 + minLength: 1 + project_or_spider_id: + title: Project or spider id + description: Project id where the update will be performed + type: string + maxLength: 100 + minLength: 1 + ProxyProviderResponse: + required: + - success + type: object + properties: + success: + title: Success + type: boolean + env_vars: + description: Env vars for the instace(project, spider) + type: array + items: + $ref: '#/definitions/SpiderJobEnvVar' JobsStats: type: object properties: diff --git a/estela-web/.env.development.example b/estela-web/.env.development.example index cb84f3f0..db0fa323 100644 --- a/estela-web/.env.development.example +++ b/estela-web/.env.development.example @@ -1,2 +1,3 @@ REACT_APP_API_BASE_URL=http://localhost:8000 REGISTER_PAGE_ENABLED=true +ESTELA_PROXIES="Estela Proxy" diff --git a/estela-web/src/components/EnvVarsSettingsPage/index.tsx b/estela-web/src/components/EnvVarsSettingsPage/index.tsx index 7b2c0ba6..adf62b40 100644 --- a/estela-web/src/components/EnvVarsSettingsPage/index.tsx +++ b/estela-web/src/components/EnvVarsSettingsPage/index.tsx @@ -14,7 +14,7 @@ import { ApiProjectsSpidersUpdateRequest, } from "../../services/api"; import { invalidDataNotification } from "../../shared"; -import { handleInvalidDataError } from "../../utils"; +import { handleInvalidDataError, getFilteredEnvVars } from "../../utils"; interface ProjectEnvVar { projectId: string; @@ -371,7 +371,7 @@ export const EnvVarsSetting: React.FC = ({ projectId, spiderId, e Environment variables will be set to all jobs in this project.

- {envVars.map((envVar: SpiderJobEnvVar, id: number) => + {getFilteredEnvVars(envVars).map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( <> diff --git a/estela-web/src/components/ProxySettingsPage/ProxyForm.tsx b/estela-web/src/components/ProxySettingsPage/ProxyForm.tsx new file mode 100644 index 00000000..abc21128 --- /dev/null +++ b/estela-web/src/components/ProxySettingsPage/ProxyForm.tsx @@ -0,0 +1,152 @@ +import React, { useEffect, useState } from "react"; +import { Button, Input, Form } from "antd"; + +import "./styles.scss"; +import { SpiderJobEnvVar } from "../../services/api"; +import { ProjectEnvVar } from "./types"; + +interface FormType { + type: string; + closeModal: () => void; +} + +interface ProxyFormProps extends ProjectEnvVar, FormType {} + +export const ProxyForm: React.FC = ({ envVars, setEnvVars, type, closeModal }) => { + const propertiesToFind: Array = [ + "ESTELA_PROXY_URL", + "ESTELA_PROXY_PORT", + "ESTELA_PROXY_USER", + "ESTELA_PROXY_PASS", + "ESTELA_PROXY_NAME", + ]; + + const [activeUpdateButton, setActiveUpdateButton] = useState(false); + // Initialize the state with the filtered envVarsData + const [proxyEnvVars, setProxyEnvVars] = useState( + envVars.filter((envVar) => propertiesToFind.includes(envVar.name)), + ); + + // Update proxyEnvVars whenever envVarsData changes + // useEffect(() => { + // const filteredProxyEnvVars = envVars.filter((envVar) => propertiesToFind.includes(envVar.name)); + // setProxyEnvVars(filteredProxyEnvVars); + // setEnvVars(envVars); + // }, [envVars]); + + useEffect(() => { + if (validProxySettings()) { + setActiveUpdateButton(true); + } else { + setActiveUpdateButton(false); + } + }, [proxyEnvVars]); + + const getProxyValue = (envVarName: string): string => { + const proxyNameObj = envVars.find((obj) => obj.name === envVarName); + return proxyNameObj ? proxyNameObj.value : ""; + }; + + const validProxySettings = (): boolean => { + return propertiesToFind.every((property) => + proxyEnvVars.some((obj) => obj.name === property && obj.value != ""), + ); + }; + + const handleChangeProxy = (e: React.ChangeEvent): void => { + const proxyEnvVar = { + name: e.target.name, + value: e.target.value, + masked: e.target.name !== "ESTELA_PROXY_NAME" ? false : false, + }; + const nameExists = proxyEnvVars.some((envVar) => envVar.name === proxyEnvVar.name); + + // If the name exists, update the value; otherwise, add the new element + const updatedProxyEnvVars = nameExists + ? proxyEnvVars.map((envVar) => (envVar.name === proxyEnvVar.name ? proxyEnvVar : envVar)) + : [...proxyEnvVars, proxyEnvVar]; + + setProxyEnvVars(updatedProxyEnvVars); + }; + + const updateEnvVars = (): void => { + setEnvVars([...envVars, ...proxyEnvVars]); + closeModal(); + }; + + return ( +
+
+ + + + + + + + + + + + + + + +
+
+ + +
+
+ ); +}; diff --git a/estela-web/src/components/ProxySettingsPage/index.tsx b/estela-web/src/components/ProxySettingsPage/index.tsx new file mode 100644 index 00000000..b9cb9173 --- /dev/null +++ b/estela-web/src/components/ProxySettingsPage/index.tsx @@ -0,0 +1,296 @@ +/* eslint-disable */ +import React, { useEffect, useState } from "react"; +import { Row, Space, Button, Tag, Popover, Modal } from "antd"; + +import "./styles.scss"; +import Menu from "../../assets/icons/menu.svg"; +import { ApiService } from "../../services"; + +import { ProxyForm } from "./ProxyForm"; +import { ProjectEnvVar, ProxyTagProps } from "./types"; +import { handleInvalidDataError, mergeArrays } from "../../utils"; +import { ESTELA_PROXIES } from "../../constants"; +import { Content } from "antd/lib/layout/layout"; + +interface estelaProxyProps { + name: string; + description: string; + id: number; +} + +export const ProxySettings: React.FC = ({ + envVars, + setEnvVars, +}) => { +// const [shouldDeleteEnvVars, setShouldDeleteEnvVars] = useState(false); + const [openProxyUserModal, setOpenProxyUserModal] = useState(false); + const [openBMCModal, setOpenBMCModal] = useState(false); + const [estelaProxies, setEstelaProxies] = useState([]); + + const apiService = ApiService(); + + const ProxyTag: React.FC = ({ children, id }) => { + const [openEditModal, setOpenEditModal] = useState(false); + const [openDeleteModal, setOpenDeleteModal] = useState(false); + const [renderTag, setRenderTag] = useState(true); + const [openPopover, setOpenPopover] = useState(false); + + const SettingContent = ( +
+ + Edit Proxy Settings

} + onCancel={() => setOpenEditModal(false)} + footer={null} + > +
+ + setOpenEditModal(false)} + setEnvVars={setEnvVars} + > + +
+ + + CONFIRM ACTION

} + onCancel={() => setOpenDeleteModal(false)} + footer={null} + > + <> +

+ Are you sure you want to delete this environment variable +

+
+ + +
+ + +
+ ); + + const tagStyle = + "w-96 h-24 flex items-center justify-between text-base pr-2 py-1 pl-3 border-estela-blue-low text-estela-blue-full bg-estela-blue-low hover:border-estela-blue-full rounded-xl"; + + return ( + <> + {renderTag && ( + +
+ {children} + You are using your own proxy. +
+
+ setOpenPopover(open)} + open={openPopover} + className="padding-0 rounded-lg" + content={SettingContent} + trigger="click" + showArrow={false} + placement="right" + > + + +
+ + )} + + ); + }; + + const getProxyValue = (envVarName: string): string => { + const proxyNameObj = envVars.find((obj) => obj.name === envVarName); + return proxyNameObj ? proxyNameObj.value : ""; + }; + const checkIfProxyExist = (): boolean => { + const propertiesToFind: Array = [ + "ESTELA_PROXY_NAME", + ]; + return propertiesToFind.every((property) => envVars.some((obj) => obj.name === property)); + }; + + const handleRemoveProxy = (): void => { + const propertiesToFind: Array = [ + "ESTELA_PROXY_URL", + "ESTELA_PROXY_PORT", + "ESTELA_PROXY_USER", + "ESTELA_PROXY_PASS", + "ESTELA_PROXY_NAME", + "ESTELA_PROXIES_ENABLED", + ]; + const filteredEnvVars = envVars.filter((obj) => !propertiesToFind.includes(obj.name)); + setEnvVars(filteredEnvVars); + }; + + + const getEstelaProxies = (): void => { + const request = {}; + const newEstelaProxies: estelaProxyProps[] = []; + apiService.apiProxyProviderList(request).then((response: any) => { + response.results.forEach((item: any) => { + newEstelaProxies.push({ + name: item.name, + description: item.description, + id: item.proxyid, + }); + }); + setEstelaProxies(newEstelaProxies); + }); + } + + useEffect(() => { + getEstelaProxies(); + }, []); + + const useEstelaProxy = (proxyId: number): void => { + const proxySelected = estelaProxies.find((item) => item.id === proxyId); + setEnvVars([...envVars, { + name: "ESTELA_PROXY_NAME", + value: proxySelected ? proxySelected.name : "", + masked: false, + }]) + setOpenBMCModal(false); + }; + + return ( +
+ {checkIfProxyExist() ? ( + + +
+

Proxy Settings

+

+ Control and configure your proxies effortlessly. +

+ + {getProxyValue("ESTELA_PROXY_NAME")} + +
+ + + ) : ( + + +
+

Proxy Settings

+

+ Control and configure your proxies effortlessly. +

+
+
+ + + {estelaProxies.length > 0 && ( + )} + + New proxy configuration

} + onCancel={() => setOpenProxyUserModal(false)} + footer={null} + > +
+ + setOpenProxyUserModal(false)} + setEnvVars={setEnvVars} + > + +
+ + Select pre-configured Proxy

} + onCancel={() => setOpenBMCModal(false)} + footer={null} + > + {estelaProxies.map((item: any) => { + return ( + + ); + })} + +
+
+
+ + + )} +
+ ); +}; diff --git a/estela-web/src/components/ProxySettingsPage/styles.scss b/estela-web/src/components/ProxySettingsPage/styles.scss new file mode 100644 index 00000000..c2aeb3ac --- /dev/null +++ b/estela-web/src/components/ProxySettingsPage/styles.scss @@ -0,0 +1,33 @@ +$estela-blue-full: #4D47C3; +$estela-blue-low: #F6FAFD; + +.tooltip { + padding: 0px; +} + +.ant-tooltip-inner { + color: white; + border-radius: 8px; +} + +.button-container { + display: flex; + justify-content: space-between; /* Opcional: para agregar espacio entre los botones */ +} + +.button-spacing { + margin-right: 10px; /* Ajusta el valor según la separación deseada */ +} + +.estela-border-proxy:hover { + border-radius: 8px; + border: 2px solid var(--estela-blue-full, #4D47C3); + background: var(--estela-blue-low, #F6FAFD); +} + +.input-proxy-form { + height: 60px; + font-size: 14; + border-radius: 8px; + color: "gray"; +} diff --git a/estela-web/src/components/ProxySettingsPage/types.ts b/estela-web/src/components/ProxySettingsPage/types.ts new file mode 100644 index 00000000..44f85693 --- /dev/null +++ b/estela-web/src/components/ProxySettingsPage/types.ts @@ -0,0 +1,25 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +import React, { useEffect, useState } from "react"; +import { Row, Space, Button, Tag, Tooltip, Checkbox, Input, Popover, Modal, Form } from "antd"; +import type { CheckboxChangeEvent } from "antd/es/checkbox"; +import { Link } from "react-router-dom"; +import { SpiderJobEnvVar } from "../../services"; + +export interface ProjectEnvVar { + envVars: SpiderJobEnvVar[]; + setEnvVars: (envVars: SpiderJobEnvVar[]) => void; +} + +export interface ProxySettingsProps { + ESTELA_PROXY_URL: string; + ESTELA_PROXY_PORT: string; + ESTELA_PROXY_USER: string; + ESTELA_PROXY_PASS: string; + ESTELA_PROXY_NAME: string; +} + +export interface ProxyTagProps { + children: React.ReactNode; + id: number; +} diff --git a/estela-web/src/constants.ts b/estela-web/src/constants.ts index 411a304d..8920b6a7 100644 --- a/estela-web/src/constants.ts +++ b/estela-web/src/constants.ts @@ -1,2 +1,3 @@ export const API_BASE_URL = process.env.REACT_APP_API_BASE_URL; +export const ESTELA_PROXIES = process.env.ESTELA_PROXIES ? process.env.ESTELA_PROXIES : "Estela Proxy"; export const REGISTER_PAGE_ENABLED = process.env.REGISTER_PAGE_ENABLED === "true"; diff --git a/estela-web/src/pages/CronJobDetailPage/index.tsx b/estela-web/src/pages/CronJobDetailPage/index.tsx index 16b48f7c..25772c0e 100644 --- a/estela-web/src/pages/CronJobDetailPage/index.tsx +++ b/estela-web/src/pages/CronJobDetailPage/index.tsx @@ -50,7 +50,7 @@ import { SpiderCronJobUpdateDataStatusEnum, } from "../../services/api"; import { resourceNotAllowedNotification, incorrectDataNotification, Spin, PaginationItem } from "../../shared"; -import { convertDateToString } from "../../utils"; +import { convertDateToString, getFilteredEnvVars } from "../../utils"; const { Option } = Select; const { Content } = Layout; @@ -637,6 +637,12 @@ export class CronJobDetailPage extends Component { + const desiredItem: SpiderJobEnvVar = envVars.find( + (item: SpiderJobEnvVar) => item.name === "ESTELA_PROXY_NAME", + ); + return desiredItem ? desiredItem.value : ""; + }; return ( <> @@ -918,7 +924,7 @@ export class CronJobDetailPage extends ComponentEnvironment variables - {envVars.map((envVar: SpiderJobEnvVar, id) => + {getFilteredEnvVars(envVars).map((envVar: SpiderJobEnvVar, id) => envVar.masked ? ( + Proxy + + + {getProxyTag() === "" ? ( + No Proxy + ) : ( + + {getProxyTag()} + + )} + + + + Arguments diff --git a/estela-web/src/pages/CronjobCreateModal/index.tsx b/estela-web/src/pages/CronjobCreateModal/index.tsx index 8fc7f45b..a3cc0e93 100644 --- a/estela-web/src/pages/CronjobCreateModal/index.tsx +++ b/estela-web/src/pages/CronjobCreateModal/index.tsx @@ -42,6 +42,8 @@ import history from "../../history"; import { ApiService } from "../../services"; import { resourceNotAllowedNotification, invalidDataNotification, incorrectDataNotification } from "../../shared"; import { checkExternalError } from "../../defaultComponents"; +import { getFilteredEnvVars } from "../../utils"; +import { ProxySettings } from "../../components/ProxySettingsPage"; import "./styles.scss"; import Add from "../../assets/icons/add.svg"; @@ -170,6 +172,9 @@ const weekOptions = [ export default function CronjobCreateModal({ openModal, spider, projectId }: CronjobCreateModalProps) { const PAGE_SIZE = 15; const apiService = ApiService(); + const [noProxy, setNoProxy] = useState(false); + const [jobProxy, setJobProxy] = useState([]); + const [newProxyFormActivate, setNewProxyFormActivate] = useState(false); const [open, setOpen] = useState(openModal); const [countKey, setCountKey] = useState(0); const [loading, setLoading] = useState(false); @@ -492,6 +497,71 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro } }; + const handleRemoveProxy = (): void => { + setNoProxy(true); + const filteredProjectEnvVars = getFilteredEnvVars(projectEnvVars); + const filteredSpiderEnvVars = getFilteredEnvVars(spiderEnvVars); + const filteredJobEnvVars = getFilteredEnvVars(cronjobData.envVars); + const newEnvVars: EnvVarsData[] = filteredJobEnvVars.map((envVar) => { + setCountKey(countKey + 1); // Increment the key + return { + name: envVar.name, + value: envVar.value, + key: countKey, + masked: envVar.masked ? envVar.masked : false, // If 'masked' is not defined, default to false + }; + }); + setProjectEnvVars(filteredProjectEnvVars); + setSpiderEnvVars(filteredSpiderEnvVars); + setCronjobData({ ...cronjobData, envVars: newEnvVars }); + }; + + const handleJobCreateProxy = (envVars: SpiderJobEnvVar[]): void => { + const proxyEnvVars: SpiderJobEnvVar[] = getFilteredEnvVars(envVars, false); + const newEnvVars: EnvVarsData[] = proxyEnvVars.map((envVar) => { + setCountKey(countKey + 1); // Increment the key + return { + name: envVar.name, + value: envVar.value, + key: countKey, + masked: envVar.masked ? envVar.masked : false, // If 'masked' is not defined, default to false + }; + }); + setCronjobData({ ...cronjobData, envVars: [...cronjobData.envVars, ...newEnvVars] }); + setNewProxyFormActivate(false); + }; + const addNewProxy = (): void => { + setNewProxyFormActivate(true); + }; + + useEffect(() => { + const getProxyEnvVars = (): SpiderJobEnvVar[] => { + const jobProxyName = cronjobData.envVars.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value; + const spiderProxyName = spiderEnvVars.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value; + const projectProxyName = projectEnvVars.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value; + return jobProxyName + ? getFilteredEnvVars(cronjobData.envVars, false) + : spiderProxyName + ? getFilteredEnvVars(spiderEnvVars, false) + : projectProxyName + ? getFilteredEnvVars(projectEnvVars, false) + : []; + }; + const newProxyJob: SpiderJobEnvVar[] = getProxyEnvVars(); + if (newProxyJob.length > 0) { + setNoProxy(false); + setJobProxy(newProxyJob); + } else { + setNoProxy(true); + } + }, [projectEnvVars, spiderEnvVars, cronjobData]); + const getCustomExpression = (): string => { const { date, recurrence, recurrenceNum, weekDays } = crontab; let days = ""; @@ -744,9 +814,13 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro

Environment Variables

- {projectEnvVars.length > 0 ?

Project:

: <>} + {getFilteredEnvVars(projectEnvVars).length > 0 ? ( +

Project:

+ ) : ( + <> + )}
- {projectEnvVars.map((envVar: SpiderJobEnvVar, id: number) => + {getFilteredEnvVars(projectEnvVars).map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( {envVar.name} @@ -765,9 +839,13 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro
- {spiderEnvVars.length > 0 ?

Spider:

: <>} + {getFilteredEnvVars(spiderEnvVars).length > 0 ? ( +

Spider:

+ ) : ( + <> + )}
- {spiderEnvVars.map((envVar: SpiderJobEnvVar, id: number) => + {getFilteredEnvVars(spiderEnvVars).map((envVar: SpiderJobEnvVar, id: number) => envVar.masked ? ( {envVar.name} @@ -839,6 +917,48 @@ export default function CronjobCreateModal({ openModal, spider, projectId }: Cro + + New proxy configuration

} + onCancel={() => setNewProxyFormActivate(false)} + footer={null} + > + + + +

Proxy

+ {noProxy ? ( + + ) : ( + +
+
+ + proxy_name:{" "} + {jobProxy.find( + (envVar: SpiderJobEnvVar) => + envVar.name === "ESTELA_PROXY_NAME", + )?.value || "none"} + +
+
+ + )} + +

Tags

diff --git a/estela-web/src/pages/JobCreateModal/index.tsx b/estela-web/src/pages/JobCreateModal/index.tsx index f85642c5..7289c375 100644 --- a/estela-web/src/pages/JobCreateModal/index.tsx +++ b/estela-web/src/pages/JobCreateModal/index.tsx @@ -12,8 +12,10 @@ import { Project, Spider, } from "../../services/api"; +import { getFilteredEnvVars } from "../../utils"; import history from "../../history"; import { ApiService } from "../../services"; +import { ProxySettings } from "../../components/ProxySettingsPage"; import { resourceNotAllowedNotification, invalidDataNotification, incorrectDataNotification } from "../../shared"; import { checkExternalError } from "../../defaultComponents"; @@ -110,8 +112,11 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea dataStatus: spider ? spider.dataStatus : undefined, dataExpiryDays: spider ? spider.dataExpiryDays : 1, }); + const [noProxy, setNoProxy] = useState(true); + const [newProxyFormActivate, setNewProxyFormActivate] = useState(false); const [projectEnvVars, setProjectEnvVars] = useState([]); const [spiderEnvVars, setSpiderEnvVars] = useState([]); + const [jobProxy, setJobProxy] = useState([]); const [variable, setVariable] = useState({ newArgName: "", newArgValue: "", @@ -179,6 +184,9 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea }; }), ); + // setProxyName( + // envVars.find((envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME")?.value || "", + // ); }, (error: unknown) => { error; @@ -186,7 +194,6 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea }, ); }; - const getProjectSpiders = async (page: number): Promise => { const requestParams: ApiProjectsSpidersListRequest = { pid: projectId, page, pageSize: PAGE_SIZE }; apiService.apiProjectsSpidersList(requestParams).then( @@ -435,6 +442,71 @@ export default function JobCreateModal({ openModal, spider, projectId }: JobCrea ); }; + const handleRemoveProxy = (): void => { + setNoProxy(true); + const filteredProjectEnvVars = getFilteredEnvVars(projectEnvVars); + const filteredSpiderEnvVars = getFilteredEnvVars(spiderEnvVars); + const filteredJobEnvVars = getFilteredEnvVars(jobData.envVars); + const newEnvVars: EnvVarsData[] = filteredJobEnvVars.map((envVar) => { + setCountKey(countKey + 1); // Increment the key + return { + name: envVar.name, + value: envVar.value, + key: countKey, + masked: envVar.masked ? envVar.masked : false, // If 'masked' is not defined, default to false + }; + }); + setProjectEnvVars(filteredProjectEnvVars); + setSpiderEnvVars(filteredSpiderEnvVars); + setJobData({ ...jobData, envVars: newEnvVars }); + }; + + const handleJobCreateProxy = (envVars: SpiderJobEnvVar[]): void => { + const proxyEnvVars: SpiderJobEnvVar[] = getFilteredEnvVars(envVars, false); + const newEnvVars: EnvVarsData[] = proxyEnvVars.map((envVar) => { + setCountKey(countKey + 1); // Increment the key + return { + name: envVar.name, + value: envVar.value, + key: countKey, + masked: envVar.masked ? envVar.masked : false, // If 'masked' is not defined, default to false + }; + }); + setJobData({ ...jobData, envVars: [...jobData.envVars, ...newEnvVars] }); + setNewProxyFormActivate(false); + }; + const addNewProxy = (): void => { + setNewProxyFormActivate(true); + }; + + useEffect(() => { + const getProxyEnvVars = (): SpiderJobEnvVar[] => { + const jobProxyName = jobData.envVars.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value; + const spiderProxyName = spiderEnvVars.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value; + const projectProxyName = projectEnvVars.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value; + return jobProxyName + ? getFilteredEnvVars(jobData.envVars, false) + : spiderProxyName + ? getFilteredEnvVars(spiderEnvVars, false) + : projectProxyName + ? getFilteredEnvVars(projectEnvVars, false) + : []; + }; + const newProxyJob: SpiderJobEnvVar[] = getProxyEnvVars(); + if (newProxyJob.length > 0) { + setNoProxy(false); + setJobProxy(newProxyJob); + } else { + setNoProxy(true); + } + }, [projectEnvVars, spiderEnvVars, jobData]); + return ( <> + ) : ( + +
+
+ handleRemoveProxy()} + > + proxy_name:{" "} + {jobProxy.find( + (envVar: SpiderJobEnvVar) => envVar.name === "ESTELA_PROXY_NAME", + )?.value || "none"} + +
+
+ + )} + +

Tags

diff --git a/estela-web/src/pages/JobDataPage/index.tsx b/estela-web/src/pages/JobDataPage/index.tsx index e706bfee..6fd831e4 100644 --- a/estela-web/src/pages/JobDataPage/index.tsx +++ b/estela-web/src/pages/JobDataPage/index.tsx @@ -372,7 +372,7 @@ export function JobItemsData({ projectId, spiderId, jobId }: JobsDataProps) { , J items, status, } = this.state; + const getProxyTag = (): string => { + const desiredItem = envVars.find((item) => item.name === "ESTELA_PROXY_NAME"); + return desiredItem ? desiredItem.value : ""; + }; const bandwidth: BytesMetric = formatBytes(Number(totalResponseBytes)); const [dataChartProportions, colorChartArray] = this.chartConfigs(bandwidth); const lifespanPercentage: number = Math.round(100 * (Math.log(lifespan ?? 1) / Math.log(3600))); @@ -753,7 +757,7 @@ export class JobDetailPage extends Component, J {date} - + Scheduled job @@ -770,7 +774,7 @@ export class JobDetailPage extends Component, J )} - + Tags @@ -790,13 +794,13 @@ export class JobDetailPage extends Component, J - + Environment variables - + - {envVars.map((envVar: SpiderJobEnvVar, id) => + {getFilteredEnvVars(envVars).map((envVar: SpiderJobEnvVar, id) => envVar.masked ? ( , J + + + Proxy + + + + {getProxyTag() === "" ? ( + No Proxy + ) : ( + + {getProxyTag()} + + )} + + + Arguments @@ -1244,33 +1264,36 @@ export class JobDetailPage extends Component, J

Environment Variables

- {newEnvVars.map((envVar: SpiderJobEnvVar, id: number) => - envVar.masked ? ( - + {getFilteredEnvVars(newEnvVars).map( + (envVar: SpiderJobEnvVar, id: number) => + envVar.masked ? ( + + + this.handleRemoveEnvVar(id) + } + className="environment-variables" + key={id} + > + {envVar.name} + + + ) : ( this.handleRemoveEnvVar(id)} className="environment-variables" key={id} > - {envVar.name} + {envVar.name}: {envVar.value} - - ) : ( - this.handleRemoveEnvVar(id)} - className="environment-variables" - key={id} - > - {envVar.name}: {envVar.value} - - ), + ), )} diff --git a/estela-web/src/pages/JobDetailPage/styles.scss b/estela-web/src/pages/JobDetailPage/styles.scss index 386e2b18..56824869 100644 --- a/estela-web/src/pages/JobDetailPage/styles.scss +++ b/estela-web/src/pages/JobDetailPage/styles.scss @@ -6,6 +6,14 @@ padding: 1px 4px; } +.proxy { + border: 1px solid #4D47C3; + border-radius: 6px; + background-color: white; + color: #4D47C3 !important; + padding: 1px 4px; +} + .tooltip { padding: 0px; } diff --git a/estela-web/src/pages/ProjectSettingsPage/index.tsx b/estela-web/src/pages/ProjectSettingsPage/index.tsx index fdfd31be..5708478d 100644 --- a/estela-web/src/pages/ProjectSettingsPage/index.tsx +++ b/estela-web/src/pages/ProjectSettingsPage/index.tsx @@ -4,6 +4,7 @@ import type { RadioChangeEvent } from "antd"; import { RouteComponentProps, Link } from "react-router-dom"; import { EnvVarsSetting } from "../../components/EnvVarsSettingsPage"; +import { ProxySettings } from "../../components/ProxySettingsPage"; import "./styles.scss"; import history from "../../history"; @@ -124,6 +125,28 @@ export class ProjectSettingsPage extends Component, + prevState: Readonly, + ): Promise { + if (prevState.envVars !== this.state.envVars) { + this.updateProjectEnvVars(); + } + } + + updateProjectEnvVars = (): void => { + const requestData: ProjectUpdate = { + envVars: this.state.envVars, + }; + const request: ApiProjectsUpdateRequest = { + data: requestData, + pid: this.projectId, + }; + this.apiService.apiProjectsUpdate(request).then((error: unknown) => { + handleInvalidDataError(error); + }); + }; + changeName = (): void => { const requestData: ProjectUpdate = { name: this.state.name, @@ -229,6 +252,10 @@ export class ProjectSettingsPage extends Component { + this.setState({ envVars: envVars }); + }; + render(): JSX.Element { const { loaded, @@ -336,6 +363,7 @@ export class ProjectSettingsPage extends Component + Delete diff --git a/estela-web/src/pages/SpiderDetailPage/index.tsx b/estela-web/src/pages/SpiderDetailPage/index.tsx index fcc8694c..143ee279 100644 --- a/estela-web/src/pages/SpiderDetailPage/index.tsx +++ b/estela-web/src/pages/SpiderDetailPage/index.tsx @@ -17,6 +17,7 @@ import { import type { RadioChangeEvent } from "antd"; import { RouteComponentProps, Link } from "react-router-dom"; import { EnvVarsSetting } from "../../components/EnvVarsSettingsPage"; +import { ProxySettings } from "../../components/ProxySettingsPage"; import "./styles.scss"; import CronjobCreateModal from "../CronjobCreateModal"; @@ -880,6 +881,33 @@ export class SpiderDetailPage extends Component { label: "Forever", key: 7, value: 720 }, ]; + updateEnvVars = (envVars: SpiderJobEnvVar[]): void => { + this.setState({ envVars: envVars }); + }; + + updateSpiderEnvVars = (): void => { + const requestData: SpiderUpdate = { + envVars: this.state.envVars, + }; + const request: ApiProjectsSpidersUpdateRequest = { + data: requestData, + pid: this.projectId, + sid: Number(this.spiderId), + }; + this.apiService.apiProjectsSpidersUpdate(request).then((error: unknown) => { + handleInvalidDataError(error); + }); + }; + + async componentDidUpdate( + prevProps: Readonly, + prevState: Readonly, + ): Promise { + if (prevState.envVars !== this.state.envVars) { + this.updateSpiderEnvVars(); + } + } + settings = (): React.ReactNode => { const { dataStatus, dataExpiryDays, persistenceChanged, envVars } = this.state; return ( @@ -932,6 +960,7 @@ export class SpiderDetailPage extends Component envVarsData={envVars} level="spider" /> + ); diff --git a/estela-web/src/services/api/generated-api/.openapi-generator/FILES b/estela-web/src/services/api/generated-api/.openapi-generator/FILES index 21451ac3..f7cf4801 100644 --- a/estela-web/src/services/api/generated-api/.openapi-generator/FILES +++ b/estela-web/src/services/api/generated-api/.openapi-generator/FILES @@ -20,6 +20,7 @@ models/InlineResponse2005.ts models/InlineResponse2006.ts models/InlineResponse2007.ts models/InlineResponse2008.ts +models/InlineResponse2009.ts models/InlineResponse401.ts models/JobsPagination.ts models/JobsStats.ts @@ -36,6 +37,9 @@ models/ProjectJob.ts models/ProjectStats.ts models/ProjectUpdate.ts models/ProjectUsage.ts +models/ProxyProvider.ts +models/ProxyProviderResponse.ts +models/ProxyProviderUpdate.ts models/ResetPasswordConfirm.ts models/ResetPasswordRequest.ts models/Spider.ts diff --git a/estela-web/src/services/api/generated-api/apis/ApiApi.ts b/estela-web/src/services/api/generated-api/apis/ApiApi.ts index 9ca3bc7e..9aa64d02 100644 --- a/estela-web/src/services/api/generated-api/apis/ApiApi.ts +++ b/estela-web/src/services/api/generated-api/apis/ApiApi.ts @@ -60,6 +60,9 @@ import { InlineResponse2008, InlineResponse2008FromJSON, InlineResponse2008ToJSON, + InlineResponse2009, + InlineResponse2009FromJSON, + InlineResponse2009ToJSON, InlineResponse401, InlineResponse401FromJSON, InlineResponse401ToJSON, @@ -93,6 +96,15 @@ import { ProjectUsage, ProjectUsageFromJSON, ProjectUsageToJSON, + ProxyProvider, + ProxyProviderFromJSON, + ProxyProviderToJSON, + ProxyProviderResponse, + ProxyProviderResponseFromJSON, + ProxyProviderResponseToJSON, + ProxyProviderUpdate, + ProxyProviderUpdateFromJSON, + ProxyProviderUpdateToJSON, ResetPasswordConfirm, ResetPasswordConfirmFromJSON, ResetPasswordConfirmToJSON, @@ -438,6 +450,33 @@ export interface ApiProjectsUsageRequest { endDate?: string; } +export interface ApiProxyProviderCreateRequest { + data: ProxyProvider; +} + +export interface ApiProxyProviderDeleteRequest { + proxyid: number; +} + +export interface ApiProxyProviderListRequest { + page?: number; + pageSize?: number; +} + +export interface ApiProxyProviderPartialUpdateRequest { + proxyid: number; + data: ProxyProvider; +} + +export interface ApiProxyProviderReadRequest { + proxyid: number; +} + +export interface ApiProxyProviderUpdateRequest { + proxyid: number; + data: ProxyProviderUpdate; +} + export interface ApiStatsListRequest { pid: string; startDate: string; @@ -2600,6 +2639,214 @@ export class ApiApi extends runtime.BaseAPI { return await response.value(); } + /** + */ + async apiProxyProviderCreateRaw(requestParameters: ApiProxyProviderCreateRequest): Promise> { + if (requestParameters.data === null || requestParameters.data === undefined) { + throw new runtime.RequiredError('data','Required parameter requestParameters.data was null or undefined when calling apiProxyProviderCreate.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + headerParameters['Content-Type'] = 'application/json'; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/proxy_provider`, + method: 'POST', + headers: headerParameters, + query: queryParameters, + body: ProxyProviderToJSON(requestParameters.data), + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => ProxyProviderFromJSON(jsonValue)); + } + + /** + */ + async apiProxyProviderCreate(requestParameters: ApiProxyProviderCreateRequest): Promise { + const response = await this.apiProxyProviderCreateRaw(requestParameters); + return await response.value(); + } + + /** + */ + async apiProxyProviderDeleteRaw(requestParameters: ApiProxyProviderDeleteRequest): Promise> { + if (requestParameters.proxyid === null || requestParameters.proxyid === undefined) { + throw new runtime.RequiredError('proxyid','Required parameter requestParameters.proxyid was null or undefined when calling apiProxyProviderDelete.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/proxy_provider/{proxyid}`.replace(`{${"proxyid"}}`, encodeURIComponent(String(requestParameters.proxyid))), + method: 'DELETE', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.VoidApiResponse(response); + } + + /** + */ + async apiProxyProviderDelete(requestParameters: ApiProxyProviderDeleteRequest): Promise { + await this.apiProxyProviderDeleteRaw(requestParameters); + } + + /** + */ + async apiProxyProviderListRaw(requestParameters: ApiProxyProviderListRequest): Promise> { + const queryParameters: any = {}; + + if (requestParameters.page !== undefined) { + queryParameters['page'] = requestParameters.page; + } + + if (requestParameters.pageSize !== undefined) { + queryParameters['page_size'] = requestParameters.pageSize; + } + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/proxy_provider`, + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => InlineResponse2009FromJSON(jsonValue)); + } + + /** + */ + async apiProxyProviderList(requestParameters: ApiProxyProviderListRequest): Promise { + const response = await this.apiProxyProviderListRaw(requestParameters); + return await response.value(); + } + + /** + */ + async apiProxyProviderPartialUpdateRaw(requestParameters: ApiProxyProviderPartialUpdateRequest): Promise> { + if (requestParameters.proxyid === null || requestParameters.proxyid === undefined) { + throw new runtime.RequiredError('proxyid','Required parameter requestParameters.proxyid was null or undefined when calling apiProxyProviderPartialUpdate.'); + } + + if (requestParameters.data === null || requestParameters.data === undefined) { + throw new runtime.RequiredError('data','Required parameter requestParameters.data was null or undefined when calling apiProxyProviderPartialUpdate.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + headerParameters['Content-Type'] = 'application/json'; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/proxy_provider/{proxyid}`.replace(`{${"proxyid"}}`, encodeURIComponent(String(requestParameters.proxyid))), + method: 'PATCH', + headers: headerParameters, + query: queryParameters, + body: ProxyProviderToJSON(requestParameters.data), + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => ProxyProviderFromJSON(jsonValue)); + } + + /** + */ + async apiProxyProviderPartialUpdate(requestParameters: ApiProxyProviderPartialUpdateRequest): Promise { + const response = await this.apiProxyProviderPartialUpdateRaw(requestParameters); + return await response.value(); + } + + /** + */ + async apiProxyProviderReadRaw(requestParameters: ApiProxyProviderReadRequest): Promise> { + if (requestParameters.proxyid === null || requestParameters.proxyid === undefined) { + throw new runtime.RequiredError('proxyid','Required parameter requestParameters.proxyid was null or undefined when calling apiProxyProviderRead.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/proxy_provider/{proxyid}`.replace(`{${"proxyid"}}`, encodeURIComponent(String(requestParameters.proxyid))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => ProxyProviderFromJSON(jsonValue)); + } + + /** + */ + async apiProxyProviderRead(requestParameters: ApiProxyProviderReadRequest): Promise { + const response = await this.apiProxyProviderReadRaw(requestParameters); + return await response.value(); + } + + /** + * In the request we should specify spider, project or job + */ + async apiProxyProviderUpdateRaw(requestParameters: ApiProxyProviderUpdateRequest): Promise> { + if (requestParameters.proxyid === null || requestParameters.proxyid === undefined) { + throw new runtime.RequiredError('proxyid','Required parameter requestParameters.proxyid was null or undefined when calling apiProxyProviderUpdate.'); + } + + if (requestParameters.data === null || requestParameters.data === undefined) { + throw new runtime.RequiredError('data','Required parameter requestParameters.data was null or undefined when calling apiProxyProviderUpdate.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + headerParameters['Content-Type'] = 'application/json'; + + if (this.configuration && (this.configuration.username !== undefined || this.configuration.password !== undefined)) { + headerParameters["Authorization"] = "Basic " + btoa(this.configuration.username + ":" + this.configuration.password); + } + const response = await this.request({ + path: `/api/proxy_provider/{proxyid}`.replace(`{${"proxyid"}}`, encodeURIComponent(String(requestParameters.proxyid))), + method: 'PUT', + headers: headerParameters, + query: queryParameters, + body: ProxyProviderUpdateToJSON(requestParameters.data), + }); + + return new runtime.JSONApiResponse(response, (jsonValue) => ProxyProviderResponseFromJSON(jsonValue)); + } + + /** + * In the request we should specify spider, project or job + */ + async apiProxyProviderUpdate(requestParameters: ApiProxyProviderUpdateRequest): Promise { + const response = await this.apiProxyProviderUpdateRaw(requestParameters); + return await response.value(); + } + /** * Retrieve stats of all jobs in a range of time, dates must have the format YYYY-mm-dd. */ diff --git a/estela-web/src/services/api/generated-api/models/InlineResponse2009.ts b/estela-web/src/services/api/generated-api/models/InlineResponse2009.ts new file mode 100644 index 00000000..b85716ba --- /dev/null +++ b/estela-web/src/services/api/generated-api/models/InlineResponse2009.ts @@ -0,0 +1,88 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * estela API v1.0 Documentation + * estela API Swagger Specification + * + * The version of the OpenAPI document: v1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +import { + ProxyProvider, + ProxyProviderFromJSON, + ProxyProviderFromJSONTyped, + ProxyProviderToJSON, +} from './'; + +/** + * + * @export + * @interface InlineResponse2009 + */ +export interface InlineResponse2009 { + /** + * + * @type {number} + * @memberof InlineResponse2009 + */ + count: number; + /** + * + * @type {string} + * @memberof InlineResponse2009 + */ + next?: string | null; + /** + * + * @type {string} + * @memberof InlineResponse2009 + */ + previous?: string | null; + /** + * + * @type {Array} + * @memberof InlineResponse2009 + */ + results: Array; +} + +export function InlineResponse2009FromJSON(json: any): InlineResponse2009 { + return InlineResponse2009FromJSONTyped(json, false); +} + +export function InlineResponse2009FromJSONTyped(json: any, ignoreDiscriminator: boolean): InlineResponse2009 { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'count': json['count'], + 'next': !exists(json, 'next') ? undefined : json['next'], + 'previous': !exists(json, 'previous') ? undefined : json['previous'], + 'results': ((json['results'] as Array).map(ProxyProviderFromJSON)), + }; +} + +export function InlineResponse2009ToJSON(value?: InlineResponse2009 | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'count': value.count, + 'next': value.next, + 'previous': value.previous, + 'results': ((value.results as Array).map(ProxyProviderToJSON)), + }; +} + + diff --git a/estela-web/src/services/api/generated-api/models/ProxyProvider.ts b/estela-web/src/services/api/generated-api/models/ProxyProvider.ts new file mode 100644 index 00000000..f2fe745b --- /dev/null +++ b/estela-web/src/services/api/generated-api/models/ProxyProvider.ts @@ -0,0 +1,72 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * estela API v1.0 Documentation + * estela API Swagger Specification + * + * The version of the OpenAPI document: v1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface ProxyProvider + */ +export interface ProxyProvider { + /** + * A name to identify the proxy + * @type {string} + * @memberof ProxyProvider + */ + name: string; + /** + * A description for the proxy + * @type {string} + * @memberof ProxyProvider + */ + description: string; + /** + * A unique integer value identifying this proxy. + * @type {number} + * @memberof ProxyProvider + */ + readonly proxyid?: number; +} + +export function ProxyProviderFromJSON(json: any): ProxyProvider { + return ProxyProviderFromJSONTyped(json, false); +} + +export function ProxyProviderFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProxyProvider { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'name': json['name'], + 'description': json['description'], + 'proxyid': !exists(json, 'proxyid') ? undefined : json['proxyid'], + }; +} + +export function ProxyProviderToJSON(value?: ProxyProvider | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'name': value.name, + 'description': value.description, + }; +} + + diff --git a/estela-web/src/services/api/generated-api/models/ProxyProviderResponse.ts b/estela-web/src/services/api/generated-api/models/ProxyProviderResponse.ts new file mode 100644 index 00000000..7622c562 --- /dev/null +++ b/estela-web/src/services/api/generated-api/models/ProxyProviderResponse.ts @@ -0,0 +1,72 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * estela API v1.0 Documentation + * estela API Swagger Specification + * + * The version of the OpenAPI document: v1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +import { + SpiderJobEnvVar, + SpiderJobEnvVarFromJSON, + SpiderJobEnvVarFromJSONTyped, + SpiderJobEnvVarToJSON, +} from './'; + +/** + * + * @export + * @interface ProxyProviderResponse + */ +export interface ProxyProviderResponse { + /** + * + * @type {boolean} + * @memberof ProxyProviderResponse + */ + success: boolean; + /** + * Env vars for the instace(project, spider) + * @type {Array} + * @memberof ProxyProviderResponse + */ + envVars?: Array; +} + +export function ProxyProviderResponseFromJSON(json: any): ProxyProviderResponse { + return ProxyProviderResponseFromJSONTyped(json, false); +} + +export function ProxyProviderResponseFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProxyProviderResponse { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'success': json['success'], + 'envVars': !exists(json, 'env_vars') ? undefined : ((json['env_vars'] as Array).map(SpiderJobEnvVarFromJSON)), + }; +} + +export function ProxyProviderResponseToJSON(value?: ProxyProviderResponse | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'success': value.success, + 'env_vars': value.envVars === undefined ? undefined : ((value.envVars as Array).map(SpiderJobEnvVarToJSON)), + }; +} + + diff --git a/estela-web/src/services/api/generated-api/models/ProxyProviderUpdate.ts b/estela-web/src/services/api/generated-api/models/ProxyProviderUpdate.ts new file mode 100644 index 00000000..9b114ce4 --- /dev/null +++ b/estela-web/src/services/api/generated-api/models/ProxyProviderUpdate.ts @@ -0,0 +1,65 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * estela API v1.0 Documentation + * estela API Swagger Specification + * + * The version of the OpenAPI document: v1 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface ProxyProviderUpdate + */ +export interface ProxyProviderUpdate { + /** + * Spider or project + * @type {string} + * @memberof ProxyProviderUpdate + */ + level: string; + /** + * Project id where the update will be performed + * @type {string} + * @memberof ProxyProviderUpdate + */ + projectOrSpiderId: string; +} + +export function ProxyProviderUpdateFromJSON(json: any): ProxyProviderUpdate { + return ProxyProviderUpdateFromJSONTyped(json, false); +} + +export function ProxyProviderUpdateFromJSONTyped(json: any, ignoreDiscriminator: boolean): ProxyProviderUpdate { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'level': json['level'], + 'projectOrSpiderId': json['project_or_spider_id'], + }; +} + +export function ProxyProviderUpdateToJSON(value?: ProxyProviderUpdate | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'level': value.level, + 'project_or_spider_id': value.projectOrSpiderId, + }; +} + + diff --git a/estela-web/src/services/api/generated-api/models/index.ts b/estela-web/src/services/api/generated-api/models/index.ts index 9f4329b1..b789bdd3 100644 --- a/estela-web/src/services/api/generated-api/models/index.ts +++ b/estela-web/src/services/api/generated-api/models/index.ts @@ -16,6 +16,7 @@ export * from './InlineResponse2005'; export * from './InlineResponse2006'; export * from './InlineResponse2007'; export * from './InlineResponse2008'; +export * from './InlineResponse2009'; export * from './InlineResponse401'; export * from './JobsPagination'; export * from './JobsStats'; @@ -32,6 +33,9 @@ export * from './ProjectJob'; export * from './ProjectStats'; export * from './ProjectUpdate'; export * from './ProjectUsage'; +export * from './ProxyProvider'; +export * from './ProxyProviderResponse'; +export * from './ProxyProviderUpdate'; export * from './ResetPasswordConfirm'; export * from './ResetPasswordRequest'; export * from './Spider'; diff --git a/estela-web/src/utils.ts b/estela-web/src/utils.ts index ab65ab19..d95b76c7 100644 --- a/estela-web/src/utils.ts +++ b/estela-web/src/utils.ts @@ -1,4 +1,5 @@ import { invalidDataNotification } from "./shared"; +import { SpiderJobEnvVar } from "./services"; export interface BytesMetric { quantity: number; @@ -133,3 +134,37 @@ export function handleInvalidDataError(error: unknown): void { console.error("Unexpected error", error); } } + +export function getFilteredEnvVars(envVars: SpiderJobEnvVar[], filter = true): SpiderJobEnvVar[] { + const reservedEnvVars: Array = [ + "ESTELA_PROXY_URL", + "ESTELA_PROXY_PORT", + "ESTELA_PROXY_USER", + "ESTELA_PROXY_PASS", + "ESTELA_PROXY_NAME", + "ESTELA_PROXIES_ENABLED", + ]; + if (filter) { + return envVars.filter((envVar: SpiderJobEnvVar) => !reservedEnvVars.includes(envVar.name)); + } else { + return envVars.filter((envVar: SpiderJobEnvVar) => reservedEnvVars.includes(envVar.name)); + } +} + +export const mergeArrays = (array1: SpiderJobEnvVar[], array2: SpiderJobEnvVar[]): SpiderJobEnvVar[] => { + // Create a map to hold the merged values + const mergedMap = new Map(); + + // Add all items from the first array to the map + array1.forEach((item) => { + mergedMap.set(item.name, item); + }); + + // Add all items from the second array to the map, updating any that have the same name + array2.forEach((item) => { + mergedMap.set(item.name, item); // This will overwrite any existing item with the same name + }); + + // Convert the map values to an array and return + return Array.from(mergedMap.values()); +}; diff --git a/installation/Makefile b/installation/Makefile index 8e6b28bf..a3b2bbf8 100644 --- a/installation/Makefile +++ b/installation/Makefile @@ -5,6 +5,7 @@ API_POD = $$(kubectl get pod -l app=estela-django-api -o jsonpath="{.items[0].me API_IP = $$(kubectl get services -n $${NAMESPACE} estela-django-api-service --output jsonpath='{.spec.clusterIP}') LOCAL_API_IP = $$(kubectl get services -n $${NAMESPACE} estela-django-api-service --output jsonpath='{.status.loadBalancer.ingress[0].ip}') RESOURCES = db registry minio zookeeper kafka +SERVICES ?= django-api celery-worker celery-beat redis build-project PLATFORM ?= linux/$(shell uname -m) @@ -31,25 +32,16 @@ delete-resources: .PHONY: build-all-images build-all-images: - -. ./local/.env && cd $(API_DIR) && \ - docker build .. --file docker-conf/Dockerfile-django-api --tag $${LOCAL_REGISTRY}/estela-django-api:latest --platform $(PLATFORM) && \ - docker build .. --file docker-conf/Dockerfile-celery-worker --tag $${LOCAL_REGISTRY}/estela-celery-worker:latest --platform $(PLATFORM) && \ - docker build .. --file docker-conf/Dockerfile-celery-beat --tag $${LOCAL_REGISTRY}/estela-celery-beat:latest --platform $(PLATFORM) && \ - docker build . --file docker-conf/Dockerfile-redis --tag $${LOCAL_REGISTRY}/estela-redis:latest --platform $(PLATFORM) && \ - docker build .. --file docker-conf/Dockerfile-build-project --tag $${LOCAL_REGISTRY}/estela-build-project:latest --platform $(PLATFORM) - -. ./local/.env && cd $(QUEUING_DIR) && \ - docker build .. --file Dockerfile --tag $${LOCAL_REGISTRY}/estela-consumer:latest --platform $(PLATFORM) - + -. ./local/.env && for service in $(SERVICES); do \ + cd $(API_DIR) && docker build .. --file docker-conf/Dockerfile-$$service --tag $${LOCAL_REGISTRY}/estela-$$service:latest; \ + done + -. ./local/.env && cd $(QUEUING_DIR) && docker build .. --file Dockerfile --tag $${LOCAL_REGISTRY}/estela-consumer:latest .PHONY: upload-all-images upload-all-images: - -. ./local/.env && docker push $${LOCAL_REGISTRY}/estela-django-api:latest - -. ./local/.env && docker push $${LOCAL_REGISTRY}/estela-celery-beat:latest - -. ./local/.env && docker push $${LOCAL_REGISTRY}/estela-celery-worker:latest - -. ./local/.env && docker push $${LOCAL_REGISTRY}/estela-redis:latest - -. ./local/.env && docker push $${LOCAL_REGISTRY}/estela-build-project:latest - -. ./local/.env && docker push $${LOCAL_REGISTRY}/estela-consumer:latest - + -. ./local/.env && for image in $(SERVICES); do \ + docker push $${LOCAL_REGISTRY}/estela-$$image:latest; \ + done .PHONY: images images: build-all-images upload-all-images diff --git a/installation/data.json b/installation/data.json new file mode 100644 index 00000000..e69de29b diff --git a/installation/helm-chart/templates/API/api-configmap.yaml b/installation/helm-chart/templates/API/api-configmap.yaml index 028a7a27..67ae8b70 100644 --- a/installation/helm-chart/templates/API/api-configmap.yaml +++ b/installation/helm-chart/templates/API/api-configmap.yaml @@ -45,4 +45,4 @@ data: {{- end }} DJANGO_EXTERNAL_APPS: {{ .Values.DJANGO_EXTERNAL_APPS | quote }} EXTERNAL_MIDDLEWARES: {{ .Values.EXTERNAL_MIDDLEWARES | quote }} - + PROXY_PROVIDERS_TO_TRACK: {{ .Values.PROXY_PROVIDERS_TO_TRACK | quote }} diff --git a/installation/helm-chart/values.yaml.example b/installation/helm-chart/values.yaml.example index 82b933bc..a76f8034 100644 --- a/installation/helm-chart/values.yaml.example +++ b/installation/helm-chart/values.yaml.example @@ -87,6 +87,13 @@ EMAILS_TO_ALERT: "" VERIFICATION_EMAIL: "" REGISTER: "" # "True" +# Data Downloads +MAX_CLI_DOWNLOAD_CHUNK_MB: "2" +MAX_WEB_DOWNLOAD_SIZE_MB: "1024" + +# Proxies +PROXY_PROVIDERS_TO_TRACK: "" + ############ QUEUEING ############ CONSUMER_PRODUCTION: "" # "False"