From eb4fb11f7e729411e700cec4108a689a0c4b4124 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 21 Apr 2023 14:20:48 +0200 Subject: [PATCH 1/2] fix postgres sql --- main/etc/sql/postgresql.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql index 2eeb0a9a..6439270a 100644 --- a/main/etc/sql/postgresql.sql +++ b/main/etc/sql/postgresql.sql @@ -33,10 +33,10 @@ CREATE TABLE doma_idds.requests ( CONSTRAINT "REQUESTS_STATUS_ID_NN" CHECK (status IS NOT NULL) ); -CREATE INDEX "REQUESTS_STATUS_POLL_IDX" ON doma_idds.requests (status, priority, locking, updated_at, new_poll_period, update_poll_period, created_at, request_id); - CREATE INDEX "REQUESTS_STATUS_PRIO_IDX" ON doma_idds.requests (status, priority, request_id, locking, updated_at, next_poll_at, created_at); +CREATE INDEX "REQUESTS_STATUS_POLL_IDX" ON doma_idds.requests (status, priority, locking, updated_at, new_poll_period, update_poll_period, created_at, request_id); + CREATE INDEX "REQUESTS_SCOPE_NAME_IDX" ON doma_idds.requests (name, scope, workload_id); CREATE SEQUENCE doma_idds."TRANSFORM_ID_SEQ" START WITH 1 @@ -214,7 +214,7 @@ CREATE TABLE doma_idds.messages ( request_id BIGINT NOT NULL, workload_id INTEGER, transform_id INTEGER NOT NULL, - processing_id INTEGER NOT NULL, + processing_id INTEGER, num_contents INTEGER, retries INTEGER, created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, From bd36811a427f8cb5069e810bac754987b3a1428c Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Fri, 21 Apr 2023 14:21:44 +0200 Subject: [PATCH 2/2] option not to verfiy cert against IAM --- common/lib/idds/common/authentication.py | 24 +++++++++++++----------- main/lib/idds/tests/panda_test.py | 8 ++++++-- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/common/lib/idds/common/authentication.py b/common/lib/idds/common/authentication.py index d457b5e2..2bc1d405 100644 --- a/common/lib/idds/common/authentication.py +++ b/common/lib/idds/common/authentication.py @@ -45,7 +45,9 @@ def decode_value(val): return int.from_bytes(decoded, 'big') -def should_verify(): +def should_verify(no_verify=False): + if no_verify: + return False if os.environ.get('IDDS_AUTH_NO_VERIFY', None): return False return True @@ -94,23 +96,23 @@ def __init__(self, timeout=None): def get_auth_config(self, vo): ret = {'vo': vo, 'oidc_config_url': None, 'client_id': None, - 'client_secret': None, 'audience': None} + 'client_secret': None, 'audience': None, 'no_verify': True} if self.config and self.config.has_section(vo): - for name in ['oidc_config_url', 'client_id', 'client_secret', 'vo', 'audience']: + for name in ['oidc_config_url', 'client_id', 'client_secret', 'vo', 'audience', 'no_verify']: if self.config.has_option(vo, name): ret[name] = self.config.get(vo, name) return ret - def get_http_content(self, url): + def get_http_content(self, url, no_verify=False): try: - r = requests.get(url, allow_redirects=True, verify=should_verify()) + r = requests.get(url, allow_redirects=True, verify=should_verify(no_verify)) return r.content except Exception as error: return False, 'Failed to get http content for %s: %s' (str(url), str(error)) def get_endpoint_config(self, auth_config): - content = self.get_http_content(auth_config['oidc_config_url']) + content = self.get_http_content(auth_config['oidc_config_url'], no_verify=auth_config['no_verify']) endpoint_config = json.loads(content) # ret = {'token_endpoint': , 'device_authorization_endpoint': None} return endpoint_config @@ -134,7 +136,7 @@ def get_oidc_sign_url(self, vo): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, - verify=should_verify(), + verify=should_verify(auth_config['no_verify']), headers=headers) if result is not None: @@ -179,7 +181,7 @@ def get_id_token(self, vo, device_code, interval=5, expires_in=60): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, - verify=should_verify(), + verify=should_verify(auth_config['no_verify']), headers=headers) if result is not None: if result.status_code == HTTP_STATUS_CODE.OK and result.text: @@ -211,7 +213,7 @@ def refresh_id_token(self, vo, refresh_token): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, - verify=should_verify(), + verify=should_verify(auth_config['no_verify']), headers=headers) if result is not None: @@ -226,7 +228,7 @@ def refresh_id_token(self, vo, refresh_token): except Exception as error: return False, 'Failed to refresh oidc token: ' + str(error) - def get_public_key(self, token, jwks_uri): + def get_public_key(self, token, jwks_uri, no_verify=False): headers = jwt.get_unverified_header(token) if headers is None or 'kid' not in headers: raise jwt.exceptions.InvalidTokenError('cannot extract kid from headers') @@ -262,7 +264,7 @@ def verify_id_token(self, vo, token): # discovery_endpoint = auth_config['oidc_config_url'] return False, "The audience %s of the token doesn't match vo configuration(client_id: %s)." % (audience, auth_config['client_id']), None - public_key = self.get_public_key(token, endpoint_config['jwks_uri']) + public_key = self.get_public_key(token, endpoint_config['jwks_uri'], no_verify=auth_config['no_verify']) # decode token only with RS256 if 'iss' in decoded_token and decoded_token['iss'] and decoded_token['iss'] != endpoint_config['issuer'] and endpoint_config['issuer'].startswith(decoded_token['iss']): # iss is missing the last '/' in access tokens diff --git a/main/lib/idds/tests/panda_test.py b/main/lib/idds/tests/panda_test.py index 7830b869..5fc4e6c5 100644 --- a/main/lib/idds/tests/panda_test.py +++ b/main/lib/idds/tests/panda_test.py @@ -6,6 +6,9 @@ os.environ['PANDA_URL'] = 'http://pandaserver-doma.cern.ch:25080/server/panda' os.environ['PANDA_URL_SSL'] = 'https://pandaserver-doma.cern.ch:25443/server/panda' +os.environ['PANDA_URL'] = 'http://rubin-panda-server-dev.slac.stanford.edu:80/server/panda' +os.environ['PANDA_URL_SSL'] = 'https://rubin-panda-server-dev.slac.stanford.edu:8443/server/panda' + from pandaclient import Client # noqa E402 """ @@ -27,7 +30,6 @@ print(f._attributes) print(f.values()) print(f.type) -""" jediTaskID = 10517 # 10607 jediTaskID = 146329 @@ -61,6 +63,7 @@ print(len(ret[1])) ret_jobs = ret_jobs + ret[1] print(len(ret_jobs)) +""" # sys.exit(0) @@ -105,7 +108,8 @@ # task_ids = [150607, 150619, 150649, 150637, 150110, 150111] # task_ids = [150864, 150897, 150910] # task_ids = [151114, 151115] -task_ids = [i for i in range(151444, 151453)] +# task_ids = [i for i in range(151444, 151453)] +task_ids = [i for i in range(45, 53)] # task_ids = [] for task_id in task_ids: print("Killing %s" % task_id)