From 900681722659d0b5edbcbb4148670f761661b5ef Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Fri, 13 Dec 2024 14:18:33 -0500 Subject: [PATCH] Add support for ART Local tests in the APIs --- core/art/constants.py | 3 + core/art/modelsART.py | 1 + core/art/views.py | 152 ++++++++++++++++++++++++++++-------------- 3 files changed, 107 insertions(+), 49 deletions(-) diff --git a/core/art/constants.py b/core/art/constants.py index 9ecadf01..091f1e7f 100644 --- a/core/art/constants.py +++ b/core/art/constants.py @@ -9,6 +9,8 @@ CACHE_TIMEOUT_MINUTES = 15 RETENTION_PERIOD_DAYS = 180 # 6 months EOS_PREFIX = 'https://atlas-art-data.web.cern.ch/atlas-art-data/grid-output/' +EOS_PREFIX_LOCAL = 'https://atlas-art-data.web.cern.ch/atlas-art-data/local-output/' +INITIAL_LOCAL_ID = 1 # dicts DATETIME_FORMAT = MappingProxyType({ @@ -118,3 +120,4 @@ "egammaValidation": "Reconstruction/egamma/egammaValidation/test" }) +AUTHORIZED_HOSTS = ['lxplus938', 'lxplus939'] \ No newline at end of file diff --git a/core/art/modelsART.py b/core/art/modelsART.py index 946f0609..21c8ebae 100644 --- a/core/art/modelsART.py +++ b/core/art/modelsART.py @@ -28,6 +28,7 @@ class ARTTests(models.Model): attemptnr = models.DecimalField(decimal_places=0, max_digits=3, db_column='attemptnr') maxattempt = models.DecimalField(decimal_places=0, max_digits=3, db_column='maxattempt') status = models.DecimalField(decimal_places=0, max_digits=3, db_column='status') + test_type = models.CharField(max_length=1000, db_column='test_type', null=True, blank=True) class Meta: db_table = f'"{settings.DB_SCHEMA}"."art_tests"' diff --git a/core/art/views.py b/core/art/views.py index b3a729aa..26d8e575 100644 --- a/core/art/views.py +++ b/core/art/views.py @@ -5,6 +5,7 @@ import json import re import time +import socket import multiprocessing from datetime import datetime, timedelta @@ -1105,6 +1106,18 @@ def loadSubResults(request): return HttpResponse(json.dumps(data, cls=DateEncoder), content_type='application/json') +def get_client_ip(request): + # Get the 'X-Forwarded-For' header (which contains the real client IP if behind a proxy) + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + if x_forwarded_for: + # If there are multiple IP addresses in the header (because of multiple proxies), + # the first one is usually the original client IP + ip = x_forwarded_for.split(',')[0] + else: + # Otherwise, use the direct connection's IP address (REMOTE_ADDR) + ip = request.META.get('REMOTE_ADDR') + + return ip @csrf_exempt def registerARTTest(request): @@ -1131,14 +1144,47 @@ def registerARTTest(request): tarindex = None inputfileid = None gitlabid = None + test_type = None # log all the req params for debug _logger.debug('[ART] registerARTtest requestParams: ' + str(request.session['requestParams'])) # Checking whether params were provided + if 'test_type' in request.session['requestParams']: + test_type = request.session['requestParams']['test_type'] + if 'requestParams' in request.session and 'pandaid' in request.session['requestParams'] and 'testname' in request.session['requestParams']: pandaid = request.session['requestParams']['pandaid'] testname = request.session['requestParams']['testname'] + elif test_type == 'local': + ''' + session_id = request.COOKIES.get('sessionid') + if session_id: + ''' + client_ip = get_client_ip(request) + try: + # Perform reverse DNS lookup to get the client's hostname + art_host = socket.gethostbyaddr(client_ip)[0] + except socket.herror: + # If reverse DNS lookup fails, return the IP address as fallback + art_host = client_ip + + if art_host.split('.')[0] in art_const.AUTHORIZED_HOSTS: + pass + else: + return JsonResponse({"error": "Invalid ART API user!"}, status=403) + + # Generate job ID for ART Local + query = {'test_type': 'local'} + localIDs = [] + localIDs.extend(ARTTests.objects.filter(**query).values('pandaid')) + if localIDs: + newIDs = sorted(localIDs, key=lambda x: x['pandaid']) + pandaid = int(newIDs[-1]['pandaid']) + 1 + else: + pandaid = art_const.INITIAL_LOCAL_ID + _logger.info("JobID: {} was generated".format(pandaid)) + testname = request.session['requestParams']['testname'] else: data = {'exit_code': -1, 'message': "There were not recieved any pandaid and testname"} _logger.warning(data['message'] + str(request.session['requestParams'])) @@ -1206,56 +1252,61 @@ def registerARTTest(request): _logger.warning(data['message'] + str(request.session['requestParams'])) return HttpResponse(json.dumps(data), status=422, content_type='application/json') - # Checking if provided pandaid exists in panda db - query = {'pandaid': pandaid} - values = ('pandaid', 'jeditaskid', 'username', 'computingsite', 'jobname') - jobs = [] - jobs.extend(CombinedWaitActDefArch4.objects.filter(**query).values(*values)) - try: - job = jobs[0] - except: - data = {'exit_code': -1, 'message': "Provided pandaid does not exists"} - _logger.warning(data['message'] + str(request.session['requestParams'])) - return HttpResponse(json.dumps(data), status=422, content_type='application/json') - - # Checking whether provided pandaid is art job - if 'username' in job and job['username'] != 'artprod': - data = {'exit_code': -1, 'message': "Provided pandaid is not art job"} - _logger.warning(data['message'] + str(request.session['requestParams'])) - return HttpResponse(json.dumps(data), status=422, content_type='application/json') - - # Preparing params to register art job - branch = concat_branch({'nightly_release_short':nightly_release_short, 'project': project, 'platform': platform}) - if 'computingsite' in job: - computingsite = job['computingsite'] - if 'jeditaskid' in job: - jeditaskid = job['jeditaskid'] - - # get files -> extract log tarball name, attempts - files = [] - fquery = {'jeditaskid': jeditaskid, 'pandaid': pandaid, 'type__in': ('pseudo_input', 'input', 'log')} - files.extend(Filestable4.objects.filter(**fquery).values('jeditaskid', 'pandaid', 'fileid', 'lfn', 'type', 'attemptnr')) - # count of attempts starts from 0, for readability change it to start from 1 - if len(files) > 0: - input_files = [f for f in files if f['type'] in ('pseudo_input', 'input')] - if len(input_files) > 0: - attemptnr = 1 + max([f['attemptnr'] for f in input_files]) - inputfileid = max([f['fileid'] for f in input_files]) - log_lfn = [f['lfn'] for f in files if f['type'] == 'log'] - if len(log_lfn) > 0: - try: - tarindex = int(re.search('.([0-9]{6}).log.', log_lfn[0]).group(1)) - except: - _logger.info('Failed to extract tarindex from log lfn') - tarindex = None - if 'jobname' in job: + # Only check ART Grid tests + if test_type and test_type == 'local': + branch = concat_branch({'nightly_release_short':nightly_release_short, 'project': project, 'platform': platform}) + attemptnr = 1 + else: + # Checking if provided pandaid exists in panda db + query = {'pandaid': pandaid} + values = ('pandaid', 'jeditaskid', 'username', 'computingsite', 'jobname') + jobs = [] + jobs.extend(CombinedWaitActDefArch4.objects.filter(**query).values(*values)) try: - gitlabid = int(re.search('.([0-9]{6,8}).', job['jobname']).group(1)) + job = jobs[0] except: - _logger.info('Failed to extract tarindex from log lfn') - gitlabid = None - _logger.info(f"""Got job-related metadata for test {pandaid}: - computingsite={computingsite}, tarindex={tarindex}, inputfileid={inputfileid}, attemptnr={attemptnr}""") + data = {'exit_code': -1, 'message': "Provided pandaid does not exists"} + _logger.warning(data['message'] + str(request.session['requestParams'])) + return HttpResponse(json.dumps(data), status=422, content_type='application/json') + + # Checking whether provided pandaid is art job + if 'username' in job and job['username'] != 'artprod': + data = {'exit_code': -1, 'message': "Provided pandaid is not art job"} + _logger.warning(data['message'] + str(request.session['requestParams'])) + return HttpResponse(json.dumps(data), status=422, content_type='application/json') + + # Preparing params to register art job + branch = concat_branch({'nightly_release_short':nightly_release_short, 'project': project, 'platform': platform}) + if 'computingsite' in job: + computingsite = job['computingsite'] + if 'jeditaskid' in job: + jeditaskid = job['jeditaskid'] + + # get files -> extract log tarball name, attempts + files = [] + fquery = {'jeditaskid': jeditaskid, 'pandaid': pandaid, 'type__in': ('pseudo_input', 'input', 'log')} + files.extend(Filestable4.objects.filter(**fquery).values('jeditaskid', 'pandaid', 'fileid', 'lfn', 'type', 'attemptnr')) + # count of attempts starts from 0, for readability change it to start from 1 + if len(files) > 0: + input_files = [f for f in files if f['type'] in ('pseudo_input', 'input')] + if len(input_files) > 0: + attemptnr = 1 + max([f['attemptnr'] for f in input_files]) + inputfileid = max([f['fileid'] for f in input_files]) + log_lfn = [f['lfn'] for f in files if f['type'] == 'log'] + if len(log_lfn) > 0: + try: + tarindex = int(re.search('.([0-9]{6}).log.', log_lfn[0]).group(1)) + except: + _logger.info('Failed to extract tarindex from log lfn') + tarindex = None + if 'jobname' in job: + try: + gitlabid = int(re.search('.([0-9]{6,8}).', job['jobname']).group(1)) + except: + _logger.info('Failed to extract tarindex from log lfn') + gitlabid = None + _logger.info(f"""Got job-related metadata for test {pandaid}: + computingsite={computingsite}, tarindex={tarindex}, inputfileid={inputfileid}, attemptnr={attemptnr}""") # extract datetime from str nightly time nightly_tag_date = None @@ -1265,6 +1316,8 @@ def registerARTTest(request): _logger.exception('Failed to parse date from nightly_tag') # Check whether the pandaid has been registered already + if test_type and test_type == 'local': + computingsite = "ART Local" if ARTTests.objects.filter(pandaid=pandaid).count() == 0: # INSERT ROW try: @@ -1289,9 +1342,10 @@ def registerARTTest(request): gitlabid=gitlabid, computingsite=computingsite, status=art_const.TEST_STATUS_INDEX['active'], + test_type=test_type, ) insertRow.save() - data = {'exit_code': 0, 'message': "Provided pandaid has been successfully registered"} + data = {'exit_code': 0, 'PandaID': pandaid, 'message': "Provided pandaid has been successfully registered"} _logger.info(data['message'] + str(request.session['requestParams'])) except Exception as e: data = {'exit_code': 0, 'message': "Failed to register test, can not save the row to DB"}