From 1322baefd6755775999a17b86189bec9e2f6df56 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 8 May 2024 15:11:28 +0200 Subject: [PATCH 01/20] Use support email as oai admin --- app/__init__.py | 2 +- app/tests/test_app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index d18254813..6e2175c10 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1499,7 +1499,7 @@ def oai_pmh(): return make_response("OAI-PMH not enabled.", 404, {'Content-Type': 'text/plain'}) oai = OAI(settings.oaipmh_repo_name, request.base_url, settings.oaipmh_repo_description, - settings.oaipmh_repo_base_identifier_url) + settings.oaipmh_repo_base_identifier_url, repo_admin_email=app.config.get('SUPPORT_EMAIL')) metadata_dict = {} for name, tosca in toscaInfo.items(): diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 48bcd3c80..79ea1bd44 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -759,7 +759,7 @@ def test_oai(self): self.assertIsNotNone(root.find(".//oaipmh:earliestDatestamp", namespace)) self.assertEqual(root.find(".//oaipmh:deletedRecord", namespace).text, "no") self.assertEqual(root.find(".//oaipmh:granularity", namespace).text, "YYYY-MM-DD") - self.assertEqual(root.find(".//oaipmh:adminEmail", namespace).text, "admin@localhost") + self.assertEqual(root.find(".//oaipmh:adminEmail", namespace).text, "support@example.com") # Test GetRecord tosca_id = "https://github.com/grycap/tosca/blob/main/templates/simple-node-disk.yml" From f757c005aff585582d37a2dca425f58d54ff5e10 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 8 May 2024 15:34:05 +0200 Subject: [PATCH 02/20] Fix ListIdentifiers --- app/oaipmh/oai.py | 5 +---- app/tests/test_app.py | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index d0af40e3c..2fff9dbdd 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -188,8 +188,6 @@ def listIdentifiers(self, root, metadata_dict, verb, metadata_prefix, from_date= root.append(error_element) else: for record_identifier in filtered_identifiers: - record_element = etree.Element('record') - header_element = etree.Element('header') identifier_element = etree.Element('identifier') identifier_element.text = f'{self.repository_indentifier_base_url}{record_identifier}' @@ -199,8 +197,7 @@ def listIdentifiers(self, root, metadata_dict, verb, metadata_prefix, from_date= header_element.append(identifier_element) header_element.append(datestamp_element) - record_element.append(header_element) - list_identifiers_element.append(record_element) + list_identifiers_element.append(header_element) root.append(list_identifiers_element) diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 79ea1bd44..87893f8c1 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -786,7 +786,7 @@ def test_oai(self): res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc') self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) - elems = root.findall(".//oaipmh:identifier", namespaces) + elems = root.findall(".//oaipmh:header", namespaces) self.assertEqual(len(elems), 1) self.assertEqual(root.find(".//oaipmh:identifier", namespaces).text, @@ -796,13 +796,13 @@ def test_oai(self): res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc&from=2020-09-10') self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) - elems = root.findall(".//oaipmh:identifier", namespaces) + elems = root.findall(".//oaipmh:header", namespaces) self.assertEqual(len(elems), 0) res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc&from=2020-09-07') self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) - elems = root.findall(".//oaipmh:identifier", namespaces) + elems = root.findall(".//oaipmh:header", namespaces) self.assertEqual(len(elems), 1) res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc&until=2020-09-07') From 066835f266a8ab851893b19838be5fe33ae7f4bb Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 8 May 2024 16:12:58 +0200 Subject: [PATCH 03/20] Fix standard --- app/oaipmh/oai.py | 38 ++++++++++++++++++++++---------------- app/tests/test_app.py | 9 ++++++++- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index 2fff9dbdd..b34d2185f 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -28,7 +28,7 @@ class OAI(): def __init__(self, repo_name, repo_base_url, repo_description, repo_identifier_base_url="https://github.com/grycap/tosca/blob/main/templates/", - earliest_datestamp="2023-03-01", datestamp_granularity="YYYY-MM-DD", + earliest_datestamp="2000-01-01", datestamp_granularity="YYYY-MM-DD", repo_admin_email="admin@localhost", ): self.repository_name = repo_name self.repository_base_url = repo_base_url @@ -89,7 +89,12 @@ def getRecord(self, root, metadata_dict, verb, identifier, metadata_prefix): return etree.tostring(root, pretty_print=True, encoding='unicode') identifier = identifier - name_identifier = identifier.split('templates/')[1] + if 'templates/' in identifier: + name_identifier = identifier.split('templates/')[1] + else: + error_element = Errors.idDoesNotExist() + root.append(error_element) + return etree.tostring(root, pretty_print=True, encoding='unicode') if name_identifier not in metadata_dict: error_element = Errors.idDoesNotExist() @@ -192,6 +197,7 @@ def listIdentifiers(self, root, metadata_dict, verb, metadata_prefix, from_date= identifier_element = etree.Element('identifier') identifier_element.text = f'{self.repository_indentifier_base_url}{record_identifier}' datestamp_element = etree.Element('datestamp') + datestamp_element.text = self.earliest_datestamp if metadata_dict[record_identifier].get('creation_date'): datestamp_element.text = metadata_dict[record_identifier].get('creation_date').strftime("%Y-%m-%d") @@ -557,6 +563,12 @@ def mapDC(self, metadata_dict): return root + def addError(self, root, error_type): + request_element = etree.SubElement(root, 'request') + request_element.text = f"{self.repository_base_url}" + error_element = error_type + root.append(error_element) + def processRequest(self, request, metadata_dict): root = self.baseXMLTree() @@ -570,6 +582,10 @@ def processRequest(self, request, metadata_dict): 'resumptionToken': 0, } + if not request.args.get('verb'): + self.addError(root, Errors.badVerb()) + return etree.tostring(root, pretty_print=True, encoding='unicode') + response_xml = None parsed_url = urlparse(request.url) query_parameters = parsed_url.query.split('&') @@ -579,23 +595,16 @@ def processRequest(self, request, metadata_dict): if key in attributes_dict: attributes_dict[key] += 1 if attributes_dict[key] > 1: - request_element = etree.SubElement(root, 'request') - request_element.text = f"{self.repository_base_url}" - error_element = Errors.badArgument() - root.append(error_element) - response_xml = etree.tostring(root, pretty_print=True, encoding='unicode') + self.addError(root, Errors.badArgument()) + return etree.tostring(root, pretty_print=True, encoding='unicode') - return response_xml # Check for unknown attributes unknown_attributes = [param.split('=')[0] for param in query_parameters if param.split('=')[0] not in attributes_dict] if unknown_attributes: - request_element = etree.SubElement(root, 'request') - request_element.text = f"{self.repository_base_url}" - error_element = Errors.badArgument() - root.append(error_element) + self.addError(root, Errors.badArgument()) response_xml = etree.tostring(root, pretty_print=True, encoding='unicode') return response_xml @@ -653,10 +662,7 @@ def processRequest(self, request, metadata_dict): handler = verb_handlers.get(verb) if handler is None: - request_element = etree.SubElement(root, 'request') - request_element.text = f"{self.repository_base_url}" - error_element = Errors.badVerb() - root.append(error_element) + self.addError(root, Errors.badVerb()) response_xml = etree.tostring(root, pretty_print=True, encoding='unicode') else: response_xml = handler() diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 87893f8c1..126e9ea4f 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -745,7 +745,7 @@ def test_oai(self): root = etree.fromstring(res.data) - self.assertEqual(root.find(".//oaipmh:error", namespace).attrib['code'], 'badArgument') + self.assertEqual(root.find(".//oaipmh:error", namespace).attrib['code'], 'badVerb') # Test Identify res = self.client.get('/oai?verb=Identify') @@ -782,6 +782,13 @@ def test_oai(self): # self.assertIsNotNone(root.find(".//dc:type", namespace_dc)) # self.assertIsNotNone(root.find(".//dc:rights", namespace_dc)) + # Test GetRecord with invalid identifier + tosca_id = 'invalid"id' + res = self.client.get('/oai?verb=GetRecord&metadataPrefix=oai_dc&identifier=%s' % tosca_id) + self.assertEqual(200, res.status_code) + root = etree.fromstring(res.data) + self.assertEqual(root.find(".//oaipmh:error", namespace).attrib['code'], 'idDoesNotExist') + # Test ListIdentifiers res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc') self.assertEqual(200, res.status_code) From 1d9708678ca19a2d051f13eafe158fe58280ec57 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 8 May 2024 16:18:23 +0200 Subject: [PATCH 04/20] Fix style --- app/oaipmh/oai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index b34d2185f..35806e2eb 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -598,7 +598,6 @@ def processRequest(self, request, metadata_dict): self.addError(root, Errors.badArgument()) return etree.tostring(root, pretty_print=True, encoding='unicode') - # Check for unknown attributes unknown_attributes = [param.split('=')[0] for param in query_parameters if param.split('=')[0] not in attributes_dict] From 90e1cd94a2a041c5e47c94271d3dbf2a27af86c3 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 8 May 2024 16:27:10 +0200 Subject: [PATCH 05/20] Add dates to listRecords --- app/oaipmh/oai.py | 74 ++++++++++++++++++------------------------- app/tests/test_app.py | 10 ++++-- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index 35806e2eb..94eabcdb8 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -134,6 +134,30 @@ def identify(self, root, verb): return etree.tostring(root, pretty_print=True, encoding='unicode') + def filterIdentifiers(self, metadata_dict, from_date, until_date): + if from_date is not None: + from_date_dt = self.isValidDate(from_date) + if until_date is not None: + until_date_dt = self.isValidDate(until_date) + + # Filter identifiers based on the date range specified by from_date and until_date + filtered_identifiers = list(metadata_dict.keys()) + + if from_date is not None or until_date is not None: + filtered_identifiers = [] + for record_identifier, record_data in metadata_dict.items(): + if record_data.get("creation_date"): + record_date = datetime.combine(record_data.get("creation_date"), datetime.min.time()) + else: + # Convert the date string to a datetime object + record_date = datetime.strptime(self.earliest_datestamp, "%Y-%m-%d") + + if (from_date is None or record_date >= from_date_dt) and \ + (until_date is None or record_date <= until_date_dt): + filtered_identifiers.append(record_identifier) + + return filtered_identifiers + def listIdentifiers(self, root, metadata_dict, verb, metadata_prefix, from_date=None, until_date=None, set_spec=None, resumption_token=None): self.addRequestElement(root, verb, metadata_prefix=metadata_prefix, from_date=from_date, @@ -163,26 +187,7 @@ def listIdentifiers(self, root, metadata_dict, verb, metadata_prefix, from_date= root.append(error_element) return etree.tostring(root, pretty_print=True, encoding='unicode') - if from_date is not None: - from_date_dt = self.isValidDate(from_date) - if until_date is not None: - until_date_dt = self.isValidDate(until_date) - - # Filter identifiers based on the date range specified by from_date and until_date - filtered_identifiers = list(metadata_dict.keys()) - - if from_date is not None or until_date is not None: - filtered_identifiers = [] - for record_identifier, record_data in metadata_dict.items(): - if record_data.get("creation_date"): - record_date = datetime.combine(record_data.get("creation_date"), datetime.min.time()) - else: - # Convert the date string to a datetime object - record_date = datetime.strptime(self.earliest_datestamp, "%Y-%m-%d") - - if (from_date is None or record_date >= from_date_dt) and \ - (until_date is None or record_date <= until_date_dt): - filtered_identifiers.append(record_identifier) + filtered_identifiers = self.filterIdentifiers(metadata_dict, from_date, until_date) # Create the ListIdentifiers element list_identifiers_element = etree.Element('ListIdentifiers') @@ -262,24 +267,7 @@ def listRecords(self, root, metadata_dict, verb, metadata_prefix, from_date=None root.append(error_element) return etree.tostring(root, pretty_print=True, encoding='unicode') - # if from_date is not None: - # from_date_dt = self.isValidDate(from_date) - # if until_date is not None: - # until_date_dt = self.isValidDate(until_date) - - # Filter identifiers based on the date range specified by from_date and until_date - # filtered_identifiers = [] - - # for record_identifier, record_data in metadata_dict.items(): - # record_date_str = record_data["date"] - # record_metadata = record_data - - # # Convert the date string to a datetime object - # record_date = datetime.strptime(record_date_str, "%Y-%m-%d") - - # if (from_date_dt is None or record_date >= from_date_dt) and \ - # (until_date_dt is None or record_date <= until_date_dt): - # filtered_identifiers.append((record_identifier, record_date_str, record_metadata)) + filtered_identifiers = self.filterIdentifiers(metadata_dict, from_date, until_date) list_records_element = etree.Element('ListRecords') @@ -288,7 +276,7 @@ def listRecords(self, root, metadata_dict, verb, metadata_prefix, from_date=None error_element = Errors.noRecordsMatch() root.append(error_element) else: - for record_name, record_metadata in metadata_dict.items(): + for record_name in filtered_identifiers: record_element = etree.Element('record') header_element = etree.Element('header') @@ -296,16 +284,16 @@ def listRecords(self, root, metadata_dict, verb, metadata_prefix, from_date=None identifier_element.text = f'{self.repository_indentifier_base_url}{record_name}' datestamp_element = etree.Element('datestamp') datestamp_element.text = 'datestamp' - if record_metadata.get('creation_date'): - datestamp_element.text = record_metadata.get('creation_date').strftime("%Y-%m-%d") + if metadata_dict[record_name].get('creation_date'): + datestamp_element.text = metadata_dict[record_name].get('creation_date').strftime("%Y-%m-%d") metadata_element = etree.Element('metadata') if metadata_prefix == 'oai_dc': - metadata_xml = self.mapDC(record_metadata) + metadata_xml = self.mapDC(metadata_dict[record_name]) if metadata_prefix == 'oai_openaire': - metadata_xml = self.mapOAIRE(record_metadata) + metadata_xml = self.mapOAIRE(metadata_dict[record_name]) # Append the generated XML to the metadata element metadata_element.append(metadata_xml) diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 126e9ea4f..4b0ef81ae 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -837,11 +837,17 @@ def test_oai(self): # Test ListRecords oai_openaire res = self.client.get('/oai?verb=ListRecords&metadataPrefix=oai_openaire') self.assertEqual(200, res.status_code) - root = etree.fromstring(res.data) - + elems = root.findall(".//oaipmh:identifier", namespaces) + self.assertEqual(len(elems), 1) self.assertEqual(root.find(".//datacite:creatorName", namespaces).text, "Miguel Caballer") + res = self.client.get('/oai?verb=ListRecords&metadataPrefix=oai_dc&until=2020-09-07') + self.assertEqual(200, res.status_code) + root = etree.fromstring(res.data) + elems = root.findall(".//oaipmh:identifier", namespaces) + self.assertEqual(len(elems), 0) + # Test ListMetadataFormats res = self.client.get('/oai?verb=ListMetadataFormats') self.assertEqual(200, res.status_code) From 7ca33dad3751ce8ccaefde5cda6976c63e6a6b20 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 8 May 2024 16:47:25 +0200 Subject: [PATCH 06/20] return noRecordsMatch --- app/oaipmh/oai.py | 6 ++++-- app/tests/test_app.py | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index 94eabcdb8..cc5784bed 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -192,10 +192,11 @@ def listIdentifiers(self, root, metadata_dict, verb, metadata_prefix, from_date= # Create the ListIdentifiers element list_identifiers_element = etree.Element('ListIdentifiers') - if not metadata_dict: + if not metadata_dict or filtered_identifiers == []: # If the metadata_dict is empty, add a noRecordsMatch error error_element = Errors.noRecordsMatch() root.append(error_element) + return etree.tostring(root, pretty_print=True, encoding='unicode') else: for record_identifier in filtered_identifiers: header_element = etree.Element('header') @@ -271,10 +272,11 @@ def listRecords(self, root, metadata_dict, verb, metadata_prefix, from_date=None list_records_element = etree.Element('ListRecords') - if not metadata_dict: + if not metadata_dict or filtered_identifiers == []: # If the metadata_dict dictionary is empty error_element = Errors.noRecordsMatch() root.append(error_element) + return etree.tostring(root, pretty_print=True, encoding='unicode') else: for record_name in filtered_identifiers: record_element = etree.Element('record') diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 4b0ef81ae..73e28b783 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -803,8 +803,7 @@ def test_oai(self): res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc&from=2020-09-10') self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) - elems = root.findall(".//oaipmh:header", namespaces) - self.assertEqual(len(elems), 0) + self.assertEqual(root.find(".//oaipmh:error", namespace).attrib['code'], 'noRecordsMatch') res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc&from=2020-09-07') self.assertEqual(200, res.status_code) @@ -815,8 +814,7 @@ def test_oai(self): res = self.client.get('/oai?verb=ListIdentifiers&metadataPrefix=oai_dc&until=2020-09-07') self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) - elems = root.findall(".//oaipmh:identifier", namespaces) - self.assertEqual(len(elems), 0) + self.assertEqual(root.find(".//oaipmh:error", namespace).attrib['code'], 'noRecordsMatch') # Test ListRecords oai_dc res = self.client.get('/oai?verb=ListRecords&metadataPrefix=oai_dc') @@ -845,8 +843,7 @@ def test_oai(self): res = self.client.get('/oai?verb=ListRecords&metadataPrefix=oai_dc&until=2020-09-07') self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) - elems = root.findall(".//oaipmh:identifier", namespaces) - self.assertEqual(len(elems), 0) + self.assertEqual(root.find(".//oaipmh:error", namespace).attrib['code'], 'noRecordsMatch') # Test ListMetadataFormats res = self.client.get('/oai?verb=ListMetadataFormats') From 54cbc44c455354aecb6e7df6cf7739b072902325 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 9 May 2024 09:07:54 +0200 Subject: [PATCH 07/20] Fix OAI POST --- app/__init__.py | 3 ++- app/tests/test_app.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 6e2175c10..68e8330c2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -66,7 +66,7 @@ def create_app(oidc_blueprint=None): else: key = None cred = DBCredentials(settings.db_url, key) - CSRFProtect(app) + csrf = CSRFProtect(app) infra = Infrastructures(settings.db_url) im = InfrastructureManager(settings.imUrl, settings.imTimeout) ssh_key = SSHKey(settings.db_url) @@ -1494,6 +1494,7 @@ def manage_vault_info(): return redirect(url_for('manage_creds')) @app.route('/oai', methods=['GET', 'POST']) + @csrf.exempt def oai_pmh(): if not settings.oaipmh_repo_name: return make_response("OAI-PMH not enabled.", 404, {'Content-Type': 'text/plain'}) diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 73e28b783..7cc5bbc3c 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -761,6 +761,10 @@ def test_oai(self): self.assertEqual(root.find(".//oaipmh:granularity", namespace).text, "YYYY-MM-DD") self.assertEqual(root.find(".//oaipmh:adminEmail", namespace).text, "support@example.com") + # Test Identify + res = self.client.post('/oai?verb=Identify') + self.assertEqual(200, res.status_code) + # Test GetRecord tosca_id = "https://github.com/grycap/tosca/blob/main/templates/simple-node-disk.yml" res = self.client.get('/oai?verb=GetRecord&metadataPrefix=oai_dc&identifier=%s' % tosca_id) From f832beb0f661ba6171e56156c7fdeb79e61d0b2f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 9 May 2024 09:28:42 +0200 Subject: [PATCH 08/20] Fix OAI POST --- app/oaipmh/oai.py | 23 +++++++---------------- app/tests/test_app.py | 4 +++- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index cc5784bed..591a7cd63 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -598,22 +598,13 @@ def processRequest(self, request, metadata_dict): return response_xml - if request.method == 'GET': - verb = request.args.get('verb') - metadata_prefix = request.args.get('metadataPrefix') - identifier = request.args.get('identifier') - from_date = request.args.get('from') - until_date = request.args.get('until') - set_spec = request.args.get('set') - resumption_token = request.args.get('resumptionToken') - else: - verb = request.form.get('verb') - metadata_prefix = request.form.get('metadataPrefix') - identifier = request.form.get('identifier') - from_date = request.form.get('from') - until_date = request.form.get('until') - set_spec = request.form.get('set') - resumption_token = request.form.get('resumptionToken') + verb = request.args.get('verb', request.form.get('verb')) + metadata_prefix = request.args.get('metadataPrefix', request.form.get('metadataPrefix')) + identifier = request.args.get('identifier', request.form.get('identifier')) + from_date = request.args.get('from', request.form.get('from')) + until_date = request.args.get('until', request.form.get('until')) + set_spec = request.args.get('set', request.form.get('set')) + resumption_token = request.args.get('resumptionToken', request.form.get('resumptionToken')) # Create a dictionary mapping verbs to functions verb_handlers = { diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 7cc5bbc3c..829f0859b 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -761,9 +761,11 @@ def test_oai(self): self.assertEqual(root.find(".//oaipmh:granularity", namespace).text, "YYYY-MM-DD") self.assertEqual(root.find(".//oaipmh:adminEmail", namespace).text, "support@example.com") - # Test Identify + # Test Identify Post res = self.client.post('/oai?verb=Identify') self.assertEqual(200, res.status_code) + root = etree.fromstring(res.data) + self.assertEqual(root.find(".//oaipmh:repositoryName", namespace).text, "IM Dashboard") # Test GetRecord tosca_id = "https://github.com/grycap/tosca/blob/main/templates/simple-node-disk.yml" From 19e8bf7095382e9525ec3d6de95937084ad79804 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Thu, 9 May 2024 09:49:19 +0200 Subject: [PATCH 09/20] add ARC icon --- app/static/images/ARC.png | Bin 0 -> 10135 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/static/images/ARC.png diff --git a/app/static/images/ARC.png b/app/static/images/ARC.png new file mode 100644 index 0000000000000000000000000000000000000000..e7c3f25db20bbe66e3ffce57236354d769ce6dae GIT binary patch literal 10135 zcmV;ICurD-P)00Be@1^@s6|4?`V0000PbVXQnQ*UN; zcVTj606}DLVr3vnZDD6+Qe|Oed2z{QJOBVEIY~r8RCwC#olT5g#hHfBZ5o3i#x1rW zxiiXUkRm!8oDJj4f>ESIqOGujO&Yvl7LjNcMv9b>HWMjQSfr6ygaygaE=M?=*uz1=uV9vL=e|twld+Xy5Vvzs#uD9>&j&Gcf0e|4%CG2;6 zA=mOv?67bXUbG|E&xO`6meu*=N zP!AI1Thj8rBO(6kum2-m>CsEL?5p4WPYF3Jd5+sozzMZ?uJv{v$bk5(zy3)+kHHeR zVaWma>Ng+HbJsQCyL8bBb)O(3*|nu$UQho&tS~?h#u_)Y;s6}a8QKTj;xS{AjZk+5 z^BD#6R#tp`+tGl(kfs7{mOhGo9?_J2oTz4tp3=v}bR74~v#@9M^TZymXYu=aPI z{Ykn~4KDe12=(QYk;8max!RlhYe<4!d-J^4+Zuz>@e^@oTUb|H!5YIV+vGj4kpwr3 zR1wRba>+j>)WfvS0D5i7)fNxRy--*JYiikWL0V*3<~kc=GM2;+6xf#t^+2sN?(Sa- z>r7YMg9%q55!Pk`Gw1J)(rQme2J{(qq1N=IfMv$SpgeH3vy)gyl++1yu(X(wj2oTR zo{R|SUq*3uo7UMgtv>R7>1vl{wPjtsfVt~@M~>A-cKiiG?FY=Kly%-fI2Mn)kCCJ5YZ7LmAx8J6$PN1iN%*}LjJ5%1$Blv>wD4fzE^-4D!1vfC&kVKU4u z6lF=UA(Y}GiH6wU&>*wplYs%f6hC94z};pciCdTfLtYY0c8GcJcY`U#g*$%yhrc!7 z&*0+h$$)@H=gzr~pV1fW>T6HY-S!yfu9l_xcz*!fNbn)fo(`qM>0}^4W6$%h1{&E? zxZDJqFn9T0;ffpi0f4QSwV`G;7U)f<5s7=wuywj6ee>(<1h5 z^|1XBOEP8gvL1i7N^T4`3RGSV7V0qqjf)J35U1wtBgT7$YYV`Ud}ux4&;R+#cLxnLK(>if^MGn2tE`WKg;logosF9dU}m_}9PE3}|H1sy zrmH>K6GuQVZO#~SrC6Pw)p?R7u;|!Q(OSmt_4g~CYQea*-%uC7)`I zQ(k^S$I2D0vgvAby}9?fu}~KK)3HLGEL~xOU48B8L0&YA^O9QyTUUGm=sKrB%2P>1 z+F#sCGTS5dy3{r45ol1bt6j{YHaI{P*70+%_wu(ePJp;h@{O*xV;ya@ z(A!eiq$i*sR&L0!m#)Nxy7#5O)IBq~S23&YUKvYW8+|0sR(g!1r4Z8%b)wlcq<1a7 zO%o}pM}In1TnsO!PRG%(AVZ#T z_x^n0rR!a{V1?>gNAoeA#n&qx0j(SECvo(UCmj+Vz5y+*^J`DV-VcHHU*a02;_F#g z@pV`3_fDh=OokjsYto>hmYUX? zYAe3RbyV6`y!Z=s+U>qJk^}3S6o0fSj3=%0@ZIoh9lPOeTWVP7^~CMw1f-~=Zn)sW zaKqQNXX_o*D%-s?CbyfSdUC^e6hGcs>fN!q-M$HH<=EPXLhZuqWOP^-2?Q+$l7 zrm?x*zPuk~;jvfuYd^md9Y6p3xYLR!Nj$3Fyprv#VoeoCjWuZj=%nev7>Z1?)I~Fd zHidE5H&awDalf>`(v2u5;vo~T>WCwyR+=iTw(vR;N3ZEPVT^9KQR3(4L^*z6-2PY9 zren*dZfwXs)E;@N0^v7o1sYVWm$gD=mZmf4=-5qrEV2ciZSg=IW-T ztvFXC`t|T?Hlu{K{?HZFZ;Pi*8TY^T)Tk$n?=E;1SKWl1+cVo&3(Y9^1k`7HcfVC7 z%%*7x#%rce$fo$Ci0Op;2RBf++5zOZq`OW_?$OPE6MPAk}<{6p>tYU4(w+R+3l$=8sicQE|Tl^MYv(=p6JOENVI%D*NX5UT)akU&4ncg;J4Iss6uFln|}wYWvwSQ zfF8Y-?y{%1t+4JaJDJ#CNCR}eRaEZ-?+bhr^h0Ug1BlN>c!uM4I^6VA*;?0udbCy= zVBCG~b?5POSBKT53m^J`9*_fY>AD*qk3~K7fUj(=Cl!Gnt(8WB0AeE7@w4pH?PxQM z%3P2~-V^!llU)9!$l^Bm3@BL2x9$e@u z@s<49%fA_B{FwoJAFZFXWlw`1fBe~!H_-GZVAdy?$x3|P{JaNQSD96TIynbx4Ao;z zVG-qvcMlA)o$G*E(yfEcd>$x~iVUxW(SdyUU;p z$hAdxwdOt$ttn5rG>d9$ssE0BkPT@W3+v7(^R$iI zkhDSuduc*Z-g;gpDr3tf13DCC_WMb0ec+?@dZ-7~EEwpj55ifI-0wEuWN)u*f z?)U8kneP@%x!QN0d8YKDA~yh#>0*yV@urdwk|eGPx$Vt;Kw48=UaV`WlM1qb{ODnC z0kCzobw6^t*kMUqaLkjDB;E=u%qAtS_kng%eXR4AS#1GxS@J}ljV;dzW)?emsbw@Y z0+dnw#>>K!lm{U9KBaV%zhVZ=RR0 zK|#F+UWK*3Qf8?)({}~JOa(0*=zxZl4wt-Sw;^@zJ3dBzhQVto)KmNuw~YCiFjGMb z8E|x%b(#H6H=_iX;8VRO@Or@t>Q0Vt6?Z(i4P_?*a-Es-L9RQ@7B!>9|E+nwxKvPb zb)ZAllI9ej;I_ZQz3WJFdV9V4ly3IFnzqum@iYblR@rtjzNenruLdIep&bw*cw__uJORHvce_VANQwv+g!q1Y?!o+vG6B6sO?#QX1;1 ze$i=bij1JP%C<=a#?922taF}KJ_59H_Lj}>Y_xnUIL%6ynNY9G_&^;zmRMzMHLF-2 zCo5nWH&Y%7upL9|s|r@M&K~_ru6GNk-5ANT^@Q6r)F2-V?^Ud_>1v}smBk~JH>#1@ zZqfrDuD$KwXk#hYNR}-_ZB~-i*@c1|0pjMHp7k=L0=Lc;*PYwwX=iJx+iJOO#?`vA zOk3);y1$gJHszaSRTvUs7GY!C%d9r;y-v2ZBI7)6WvvsaD|Z&8t4(?7YFnP21`8+8 z7jH3E~3++@o3J~S>iDTu4HEOkE6Z~u%FiSh#G zvPK&^tL$Pm#>PswnIJ37FR1RJ*)hU6P6(s|fZ26Pa#=i$*Vt9~ddsK+1<$KdgEeZ3 z>QcJLw$Cuz8o(~Cv{~8>o>w__DBo1pY1YThLPdoxT`Xf2j$c%;oP`&9_qkt5x1p;b zpQ45gvyJQB^+`8}sj9LYU^IHr}#w)UFd` zsao~*O71t}1{pi39WS^8(Te*mSdqL*CvVWQr`G4rV757)*!R5!MLV-F#(g&kX%thN zRy@w`y(0ksQ`}~S>)KA;SbcDMv7S$E&bV%USVGxwTXf06&*9j*E z=%RqcRTq|=koysnmff6hiJ$Gt&DA~N%eM9ot=I`xy6EQg)sUesbJYoXx7N>gt6|xV z8NcVIxH+`71JAYx{7_G4DYnvYIeAUJEol+*?kvUeAO5!02K1Py_niMjboI5TYR7}N z4Jy0bHPu(nA{XkOD#)BcT?V<1$#HS9>xA7$-2k0ffAKr&*j0MKlYPZ?3 z?fA=k->wp5nNVw>5>7ue13oQYvyoq``m1U zF-PVaJN9grq_SMmF}A+Si>F;B$gZhS^nyozw`tn4If&UStbD;Yz~lXsa%P7tc*Ck#ybj5*RXG z{~$S`a7b>v3!sz9VhMCoJko8K59d}o%Tu}OL6X;fqD;UP!~Sk=A_9PFYv0tmCqJJt zLG{6aiRYo9_sPmvv6L=^>NI4v#~uRZ%|jm&fL?FC)OWnJg*UA^K89Q0M(nOHRMCyY zJQ37G|6rR||Wo9nIOjx!Nu5@;-0_$U!7f;I_Ssp-rV8Swl^7>66bp#wm@NdB{jC6X=`uW28(>Kb$zXR}39 zi{l;UfeyI|GDn3pDKB*~`pzeD_WOfei;fG_<$C*AtutT__3X4P*IQOZdNLl%P!HTv z*S6>?10NT=oTgh{qAy;4H0bKStYoFrSUSb+Do)8eJUoo=Kw@GCdG}QrcoX!*UDsBf z)_iHaLRDUWZ>OD1mqW@08xNMa?Rtp<7t^%lgq&77@%%j6fFz9cryA;g@mRM#FgwUN z@{^C{-B{m|33-^xil<#ykQv+_m@&14dJ;%X-8JO6n~!xY&2@TOY7gS+Xld1IVM2Wn z6VK(IHXy$iy_m%0=oL1m#M{lB0-GSLw78a8H#yf!dXw8;;*zBX;hVJ7-Nod0NVsDw zK4e!t>E3T~@_IZ(ju58uGz-BiQ^B=-RZeu6HUA zwx0>LH06Z4b5|V}+_d7_P50S~yWS`48lLpE>H6f2imvw&sFPCBVArZWu;P01NGlG= zJ<4(D8t-sj?{8;VY4@C@*QgVyv9@)1c!(f-jmui@WAW?ol>YdtS8h6h4*3)ov~Kdv z^^PrdvYF!m9GjL}Q$C$=z12@Q{rCyhbpP?*0Y}C=s9jiTsWM{UT<_SkdnihK+ee`0 zE+$VUCQ!02t^mhzM_sR2!0nd3>jGAGg3ON)^^OdvSJM7?96y{UL<0aEu%(VM?FQ(; zTQjd_tWW#kdZ#!ohMGkv!->Dqy0?ph5Zx}C%bQ0nwTG^EHq;`>zHaf@t~=h?I{)n*3G=l#U*u^hp1a;DPUlOnIrV+F zlV!(G@WKwsqSUM}N5>9%KDD?h&@z|1-rgMMxSWw+6hRH*xAi1p5z25>HDm(JGSL?B z+Ov)8onBq=ChNU?6JN5_neKP|L(G5=Nmc6$hIkehdHRi%l%0`o8&UrB^Mo-d0d6 z6Tg?3e%69}7EhNa$+(PFKbH2!QVZ@3*L$zX^|rCpOQxQ&^{^~8+-??|SjMYop2>mQ zy`@fZw^w?zFZ%{nTY6CR#;N{=dhfFKz_il#W0{q3^d+7)xI@GKi;baH+LhmY|AQ%YZaDR{_a_sn?m(}v5UlItP1gTP9261mO91TtTm|3 zgc%YFRkr$3-KePOgv(g9EumMITA!t+W9hO4SyfBDWcrVN=XuTsm&!QX$&@e6<_B%1 zJ2)Vrs#2Wo+Wwq1)Z#92IWjkSaCytyc$x_^`?N<5ON>HnG`hF6?(^`9Hsc*2UptbHS*r?>xQlr@9 zc7kl*W*MuArG|EBt^JIrWrB=Q+ZiaVP3PPkF^(>ac2ZV)r|QpFhMLyeXDj^{2gue+ z7c9-*a+xG+Xr=2~YFcYg<7p{!>Do}!TKi|I5d*T$+j0yuS{5x? z>Q6oztd(wJsh6zvU1xv7nc_H}W`eAWl`ih&OhniXUf z=IwHoRu0r3IC)*E&L$UohgxY{e{lEHuW-gVilpa+li$?{YE|Vj z*H(l(xZGH3Cdev4U9us`h3gG&jB7iuQa#p%rG~bWn1I&0Q}-+@o<>rK6=ZhGYb>}F z$~Z_irnuQ^`uiF~t#*#cpdPzjxY>hN+F#eZyw!Ez?l*Wbc8TrS?!@ZaQbVXs-PbRY zNwc2gY0A#iv1rtI*yD1ROpX~#SW(3q?@pmsC)qP;)}`y1bG{STrtK}Fjrt5_z=;tK zY%7lZfLei#US@mRRb+>h_JppXS+U?(JaxlP1zRlB`H9{_0l|miIqYSfGwmE|eYc5J zlR~=T-Nw`0yiv`|DNB&`>3V|;!*e)@8ZKdV3bmOu!ywWPZ);LM&RDt+*SjaEQ*L)F z$+0diHNZy4yE)zP_FTs-OHC_n>*eJ4AUkTQyU$Og-0r#JCGr|-r7hD9=TW%ry54O6 zTMZ&xA*AV2;2`C8Z*}r_I*A!#cVbJs;c{`EJGtn3_bi(_@1>h%$7|GgExX-YZM)rF zTWXyRpGj-;z(XubY0*;C^=>zo?&Yg6F$7oIcLm#1JP#t!<3qg`3*BVfZlVdBir zQq%Rew2+RMu`r)g;pMce^=L1kR$yN-U6JEI{4K#wTbnE^P1id&_W*bM?sLD2zP9`S zR(qzq1RC?%P%nwGBL>06n@7Un@bGX=e7K9uAO_-#mmg*GJi6YM+-$hHWsL;wZw~j# z3bfkZkhr;E9;fz4Uv$Ek%L^mGMlYT%?OtAe?J0sSbnnj>OKYrhY+m^uNsi*uPptb^ zEiAE`k5@mF?V+w;{&lr^sDMsox}B}5tvJ0v@f)AXJFgfJ>AD2Ih7AjpE}C$DRU)OS7o3u~a&JdOx;w6xR}G}u`iID$@)9R;U< zm;aF`kAj^8bf0U{L$K5YJIb#04z?MB2S8_n?K#krCEu>);o;%QC~k+eAt!Y}S@C;b z`ty>dMmF|~A8iE5G$3kixvC{0D~>S0xJKRq?Uki2v)E%VP|4go9cb3O)%E8#W9&1} zdJnXhP^ZA&3kB@#ZI%J{^~QEZ|Ia(0`wFzTP;0Qo<7}kXv32EH){@UCpw7tdhG?n# zTDEE+tV2KME6`p;ZNUA^64>WnAKmV*ytYN5ZQw{9yzVQ|-uJv-8N|ho2Xae)j$JUh z>xUz8y+gFpp7)5~8UoORfm(r$;&}8sliN0NM?Zf4_oGa-SrY!(YDc$Ep-P+$>RF14{0L?p>FPyUr=z*Ot`TGhGbeNaObQ|OP~Xfr(5(= zW(wvDA-LLv`Y7BVsy(vnI^Kqm$O7DLyX62nuh8OF@K87QQgqz>%vg#@2z7I~5pPdT zxJ|dbjczMNJIO*{%4)DP4cBWLTHH*i?cu&`z`f*_quUCy<;a?2ow@Qr3wOR~(?ame zSyw-gZoKzaRX1D1e3`S?n8zG~d>tMho{Hjx3wDC7ng53gh;E;@9ZIml1oh~7`p5Ra z1wGaXbM8L(dRZMfvf1b`yA!&LO{km0jROX@qj))NeaXUp@{x2MaRh2nqrmj>v`g`1 z;78RWEGcOVyss?tZm7vGLftLkPbj!Iv*5%Dm4Ms?9Pv2Z_dLPsUwiY#@Ft%KV-U_1 z^d^Hi(=w%)#aQD?i zT4Sys6KblSRgVm}-GqJG;-8wQ;7z_p>r1F9cQ=4jmahCh*McdEq9}@@D2k#eilQir zq9}@@D2k#eilQirq9|&tYNbr{^XI>gvczc=mnKd(vc^_pM~vhDiO0h|06pe8SxYMw2Jbh@nyrm|Lhh;e zN;MPqdoL7pT>%}!oUE)>i-Ch{GglWW6m5z39SN7_8BUSrVT~Be^)}t{K<8wumO9kI z+oHv8O*<;b(3S{vU8vQh4tekZ_Afb2S_8YWE6zFD0O|nNS#7n= zclun{ZE-BmM7N7QY&FHtnd_>yV>H$|**Vnej#%X)p^|8`gNm$9q%QCk@jhWO<`v?z zE}=B*M94`D%2#`M)iwU3~9u$UwIN^=t%_@nT1lD@0XE zWKBC7_gUx{VI>v2RH@b7AJH9gUcm)t7Q8r?wUt#aBj7pNnx(FiUEB|J%vku4@AXv4 zGOyd-NI56_gPNlH5a(n=sA+{LdPXCInxZH|O;Hq~rYMR~Qxrw0DT+{2R1;M#6yb;w zYRZ3AS$Xbs!{ia5sgQ!}k8H!L-o@+a_&Ypyt8nb2< zTSrlZn(`cI9yUm*sS&x{1iD8x$$qc5e3GF{t#c~0u6^;?s&NtrYCup^lqqb`n5p=gFcWIJs!?}a5z&paZu7pm1;}`??2}MaDb2b(^->+wzE$H# z_tbGZ)PTh7cK}k^=UI&(-Q}Txgqj*3*fWAns42g}UJz_TO^r)af=#HYaarq63HG?v zB>Rm?F_vv~D3;g=32Xr;z@FE5-vE2T-B#m-`d}<2fMSv!glMrFfIY|05^TEP)OcL+ zH3jy=97RG+5p0T3Q-Om$=M^H<6fHKPrYHmK%_!bJSqJQs9lWgl3g5fIt4OG+qhNoi zEVdV5tJh=uTGr?>5NawLq$@SS-ss3;S2?0*hfq_L|6o^vdNcH@H$qK$54Jkdqe^TY zbU7=VBv7Sb8)ZPeg^{gV;+sew%M}C5NgUT*sWE5;WdcI*P}Q)KMNV~bn(f&LIPq;gEp(+!r8v>Dq}ex1K6t9 zTh(F@9Ep6gtwZ%x-#l4}A%5?vC@xl+j-2DAIr%RE1^`lbFAPPX17`pL002ovPDHLk FV1i3ASO)+A literal 0 HcmV?d00001 From 435685a0f27512b9529d32fbd6e16208afc24677 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 9 May 2024 10:00:32 +0200 Subject: [PATCH 10/20] Fix OAI POST --- app/oaipmh/oai.py | 27 +++++++++++++-------------- app/tests/test_app.py | 5 +++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index 591a7cd63..c9ea6b2dd 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -572,25 +572,24 @@ def processRequest(self, request, metadata_dict): 'resumptionToken': 0, } - if not request.args.get('verb'): + verb = request.values.get('verb') + + if not verb: self.addError(root, Errors.badVerb()) return etree.tostring(root, pretty_print=True, encoding='unicode') response_xml = None - parsed_url = urlparse(request.url) - query_parameters = parsed_url.query.split('&') - for param in query_parameters: - key = param.split('=')[0] + for key in list(request.values.keys()): if key in attributes_dict: attributes_dict[key] += 1 + # Check for duplicate attributes if attributes_dict[key] > 1: self.addError(root, Errors.badArgument()) return etree.tostring(root, pretty_print=True, encoding='unicode') # Check for unknown attributes - unknown_attributes = [param.split('=')[0] for param in query_parameters - if param.split('=')[0] not in attributes_dict] + unknown_attributes = [key for key in list(request.values.keys()) if key not in attributes_dict] if unknown_attributes: self.addError(root, Errors.badArgument()) @@ -598,13 +597,13 @@ def processRequest(self, request, metadata_dict): return response_xml - verb = request.args.get('verb', request.form.get('verb')) - metadata_prefix = request.args.get('metadataPrefix', request.form.get('metadataPrefix')) - identifier = request.args.get('identifier', request.form.get('identifier')) - from_date = request.args.get('from', request.form.get('from')) - until_date = request.args.get('until', request.form.get('until')) - set_spec = request.args.get('set', request.form.get('set')) - resumption_token = request.args.get('resumptionToken', request.form.get('resumptionToken')) + verb = request.values.get('verb') + metadata_prefix = request.values.get('metadataPrefix') + identifier = request.values.get('identifier') + from_date = request.values.get('from') + until_date = request.values.get('until') + set_spec = request.values.get('set') + resumption_token = request.values.get('resumptionToken') # Create a dictionary mapping verbs to functions verb_handlers = { diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 829f0859b..cc9fa0a68 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -761,8 +761,9 @@ def test_oai(self): self.assertEqual(root.find(".//oaipmh:granularity", namespace).text, "YYYY-MM-DD") self.assertEqual(root.find(".//oaipmh:adminEmail", namespace).text, "support@example.com") - # Test Identify Post - res = self.client.post('/oai?verb=Identify') + # Test Identify Post with body params + res = self.client.post('/oai', headers={'Content-Type': 'application/x-www-form-urlencoded'}, + data="verb=Identify") self.assertEqual(200, res.status_code) root = etree.fromstring(res.data) self.assertEqual(root.find(".//oaipmh:repositoryName", namespace).text, "IM Dashboard") From ed554878d646f390671ec189ef2cebfa44b5fbf1 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 9 May 2024 10:10:36 +0200 Subject: [PATCH 11/20] Minor change --- app/oaipmh/oai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/oaipmh/oai.py b/app/oaipmh/oai.py index c9ea6b2dd..69493b3a9 100644 --- a/app/oaipmh/oai.py +++ b/app/oaipmh/oai.py @@ -597,7 +597,6 @@ def processRequest(self, request, metadata_dict): return response_xml - verb = request.values.get('verb') metadata_prefix = request.values.get('metadataPrefix') identifier = request.values.get('identifier') from_date = request.values.get('from') From 2cddc652182612e0c7a1b6719f4f93f37a4ae5ef Mon Sep 17 00:00:00 2001 From: Miguel Caballer Fernandez Date: Mon, 13 May 2024 09:56:03 +0200 Subject: [PATCH 12/20] Add cloud.egu.eu to LIP --- sites.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sites.json b/sites.json index c62dc8834..79561f666 100644 --- a/sites.json +++ b/sites.json @@ -55,7 +55,8 @@ "eosc-synergy.eu": "ddf0c468c8af4e0bbb9808bfc0288381", "worsica.vo.incd.pt": "a53ca78c534046e5b13f4537ae698411", "vo.imagine-ai.eu": "009f77df459b4a6389910e0fb20ddcaf", - "vo.ai4eosc.eu": "c61c1bb323414a248cb142eb6183d4b2" + "vo.ai4eosc.eu": "c61c1bb323414a248cb142eb6183d4b2", + "cloud.egi.eu": "6b042927bcfa466cb9eb56d3ea679987" } }, { From afcb2c9473c56b98941176b75bb909dc45d4249b Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 15 May 2024 15:04:14 +0200 Subject: [PATCH 13/20] Add ID help message: #548 --- app/templates/modal_creds.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/templates/modal_creds.html b/app/templates/modal_creds.html index b881333e6..42523d8aa 100644 --- a/app/templates/modal_creds.html +++ b/app/templates/modal_creds.html @@ -40,10 +40,11 @@
+ + Unique ID in your credentials (free-form e.g. SITE_NAME-VO_NAME). +
-
-
{% if cred_type == "EC2" %}
From d33d6d504ba17b02c496962d2a73981a18997e3d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 10:16:06 +0200 Subject: [PATCH 14/20] Improve SSH key validation: #549 --- app/__init__.py | 2 +- app/ssh_key.py | 37 +++++-------------------------------- app/tests/test_ssh_key.py | 26 ++++++++++++++++++++++---- requirements.txt | 1 + 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 4e227af55..a8b4d41d2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1431,7 +1431,7 @@ def write_ssh_key(): key = request.form['sshkey'] desc = request.form['desc'] - if key == "" or str(SSHKey.check_ssh_key(key.encode())) != "0": + if key == "" or not SSHKey.check_ssh_key(key): flash("Invaild SSH public key. Please insert a correct one.", 'warning') return redirect(url_for('get_ssh_keys')) diff --git a/app/ssh_key.py b/app/ssh_key.py index 2e5e37b89..6da8ecc8a 100644 --- a/app/ssh_key.py +++ b/app/ssh_key.py @@ -19,9 +19,7 @@ # specific language governing permissions and limitations # under the License. """Class to manage user SSH key using a DB backend.""" -import base64 -import binascii -import struct +import paramiko from app.db import DataBase @@ -78,33 +76,8 @@ def delete_ssh_key(self, userid, keyid): @staticmethod def check_ssh_key(key): - # credits to: https://gist.github.com/piyushbansal/5243418 - - array = key.split() - - # Each rsa-ssh key has 2 or 3 different strings in it, first one being - # typeofkey second one being keystring third one being username (optional). - if len(array) not in [2, 3]: - return 1 - - typeofkey = array[0] - string = array[1] - - # must have only valid rsa-ssh key characters ie binascii characters try: - data = base64.decodebytes(string) - except binascii.Error: - return 1 - - a = 4 - # unpack the contents of data, from data[:4] , it must be equal to 7 , property of ssh key . - try: - str_len = struct.unpack('>I', data[:a])[0] - except struct.error: - return 1 - - # data[4:11] must have string which matches with the typeofkey , another ssh key property. - if data[a:a + str_len] == typeofkey and int(str_len) == int(7): - return 0 - else: - return 1 + paramiko.PublicBlob.from_string(key) + except Exception as ex: + return False + return True diff --git a/app/tests/test_ssh_key.py b/app/tests/test_ssh_key.py index d0f28aaf9..7b8f5f3ab 100644 --- a/app/tests/test_ssh_key.py +++ b/app/tests/test_ssh_key.py @@ -64,9 +64,27 @@ def test_check_ssh_key(self): "+ttuEqy3SM2ZBuhD6xrpAUGrr0TrJBJnVVBKL31zFSu6GcDtVyjoYGJhM/vU9VuBrUHO+qYIrcGP7VaPSOgTSj7V3OLD7pp8kYmFP" "vLKleDSI/eiKO0nH/J6W2mGa1J6FDFaIIsLIyERdgakjvrkecfv/YfqPWkUGp1xnzNugkOug1ZMQHfuSs7Ag+kVP3TDPQoAo8u2Yy" "EwbLK/vVSFlTe5eaotfCmiltVu3UaPYM8QylCCTW7QCncE= micafer") - res = SSHKey.check_ssh_key(key.encode()) - self.assertFalse(res) + res = SSHKey.check_ssh_key(key) + self.assertTrue(res) - key = "ssh-rsa AAAAB3NzaC1yc2EAAAADA..." - res = SSHKey.check_ssh_key(key.encode()) + key = ("ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGTCowf1QVu0fi73aFWfsSnYixGeO6" + "03FxkmUtDAuBop2kNnjupKyf7QNSw6D8HJmWGjaeGUUhmL2r3PltoLjMA= micafer@some") + res = SSHKey.check_ssh_key(key) + self.assertTrue(res) + + key = ("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBC4bNaGHkWJW6xoQUmbpJaNzsVz22xsBYwAEBQkaL2A micafer@DESKTOP-6VOC4C3") + res = SSHKey.check_ssh_key(key) self.assertTrue(res) + + key = ("ssh-dss AAAAB3NzaC1kc3MAAACBAIUsxgWjdFpzAG8QJtg1ogDrWgkLXRNA+eXeB5Xq/9Z/NaJ2ZZFGpkgtyvJRInc0E+4RcetAk5zRYN" + "pefw2WRaxXtdyprpcch8O3InpatpSH9L3sIF8FnJLmX+s4V2PlanGFDBA8IvNfrV4IQvD3PoTi4OqlwJTuSMtOXTJ3NrRFAAAAFQDh/v0J" + "ma1BSBFTi4+wKfa7nhh06QAAAIAxcPrc+PomR9u+P9hIOoz8vpsqZ+V5V1Caev+Oiq/JyI4iRg0Hig5br47c6Ckb1DupqgQAD9cJGQ8Fo7" + "RCmNpdvcOmUxTCN3GDWrceCjv/d+ce1hDVPKlleQ5RNAbJr0/MULswhJb5wHq1aoHm/fnXgtAwwBMgZe+Z3ruggLt7YgAAAIAfPpAYiobe" + "ANSlTgS/tDM9nYCjXENBOcpAwXtN9qMCYxf+DCygz6Jr6CCmxrcVPTHbMq6Pjn4gKiGYnQDoulctW9zOWiX3SMc30N/ipkpPjau/ZJiQ2xX" + "PLMfH+SYlYH9O7Mh8TLfKuf9Ketp2LUWRIGyzR2SkNYM/cw3X91Tbxw== micafer@DESKTOP-6VOC4C3") + res = SSHKey.check_ssh_key(key) + self.assertTrue(res) + + key = "ssh-rsa AAAAB3NzaC1yc2EAAAADA..." + res = SSHKey.check_ssh_key(key) + self.assertFalse(res) diff --git a/requirements.txt b/requirements.txt index 031913b11..6ad296ab4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ Flask-WTF==1.2.1 hvac==2.2.0 tosca-parser==2.10.0 mysqlclient==2.2.4 +paramiko==3.4.0 From 723cef9fa0cefb6aa183a1862dd0bcb574f8da35 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 10:21:45 +0200 Subject: [PATCH 15/20] Fix style --- app/ssh_key.py | 2 +- app/tests/test_ssh_key.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/ssh_key.py b/app/ssh_key.py index 6da8ecc8a..d86dff7b7 100644 --- a/app/ssh_key.py +++ b/app/ssh_key.py @@ -78,6 +78,6 @@ def delete_ssh_key(self, userid, keyid): def check_ssh_key(key): try: paramiko.PublicBlob.from_string(key) - except Exception as ex: + except Exception: return False return True diff --git a/app/tests/test_ssh_key.py b/app/tests/test_ssh_key.py index 7b8f5f3ab..5fe8bbc6e 100644 --- a/app/tests/test_ssh_key.py +++ b/app/tests/test_ssh_key.py @@ -72,16 +72,18 @@ def test_check_ssh_key(self): res = SSHKey.check_ssh_key(key) self.assertTrue(res) - key = ("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBC4bNaGHkWJW6xoQUmbpJaNzsVz22xsBYwAEBQkaL2A micafer@DESKTOP-6VOC4C3") + key = ("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBC4bNaGHkWJW6xoQUmbpJaNzsVz22xsBYwAEBQkaL2A" + " micafer@DESKTOP-6VOC4C3") res = SSHKey.check_ssh_key(key) self.assertTrue(res) - key = ("ssh-dss AAAAB3NzaC1kc3MAAACBAIUsxgWjdFpzAG8QJtg1ogDrWgkLXRNA+eXeB5Xq/9Z/NaJ2ZZFGpkgtyvJRInc0E+4RcetAk5zRYN" - "pefw2WRaxXtdyprpcch8O3InpatpSH9L3sIF8FnJLmX+s4V2PlanGFDBA8IvNfrV4IQvD3PoTi4OqlwJTuSMtOXTJ3NrRFAAAAFQDh/v0J" - "ma1BSBFTi4+wKfa7nhh06QAAAIAxcPrc+PomR9u+P9hIOoz8vpsqZ+V5V1Caev+Oiq/JyI4iRg0Hig5br47c6Ckb1DupqgQAD9cJGQ8Fo7" - "RCmNpdvcOmUxTCN3GDWrceCjv/d+ce1hDVPKlleQ5RNAbJr0/MULswhJb5wHq1aoHm/fnXgtAwwBMgZe+Z3ruggLt7YgAAAIAfPpAYiobe" - "ANSlTgS/tDM9nYCjXENBOcpAwXtN9qMCYxf+DCygz6Jr6CCmxrcVPTHbMq6Pjn4gKiGYnQDoulctW9zOWiX3SMc30N/ipkpPjau/ZJiQ2xX" - "PLMfH+SYlYH9O7Mh8TLfKuf9Ketp2LUWRIGyzR2SkNYM/cw3X91Tbxw== micafer@DESKTOP-6VOC4C3") + key = ("ssh-dss AAAAB3NzaC1kc3MAAACBAIUsxgWjdFpzAG8QJtg1ogDrWgkLXRNA+eXeB5Xq/9Z/NaJ2ZZFGpkgtyvJRInc0E+4RcetAk" + "5zRYNpefw2WRaxXtdyprpcch8O3InpatpSH9L3sIF8FnJLmX+s4V2PlanGFDBA8IvNfrV4IQvD3PoTi4OqlwJTuSMtOXTJ3NrRFA" + "AAAFQDh/v0Jma1BSBFTi4+wKfa7nhh06QAAAIAxcPrc+PomR9u+P9hIOoz8vpsqZ+V5V1Caev+Oiq/JyI4iRg0Hig5br47c6Ckb1" + "DupqgQAD9cJGQ8Fo7RCmNpdvcOmUxTCN3GDWrceCjv/d+ce1hDVPKlleQ5RNAbJr0/MULswhJb5wHq1aoHm/fnXgtAwwBMgZe+Z3r" + "uggLt7YgAAAIAfPpAYiobeANSlTgS/tDM9nYCjXENBOcpAwXtN9qMCYxf+DCygz6Jr6CCmxrcVPTHbMq6Pjn4gKiGYnQDoulctW9z" + "OWiX3SMc30N/ipkpPjau/ZJiQ2xXPLMfH+SYlYH9O7Mh8TLfKuf9Ketp2LUWRIGyzR2SkNYM/cw3X91Tbxw==" + " micafer@DESKTOP-6VOC4C3") res = SSHKey.check_ssh_key(key) self.assertTrue(res) From f516f8cd6fd94325347b1d023d1bb55b7a1171e0 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 11:03:13 +0200 Subject: [PATCH 16/20] Implements: #550 --- app/templates/advanced_config.html | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/app/templates/advanced_config.html b/app/templates/advanced_config.html index 15c361d3e..2efda5f7a 100644 --- a/app/templates/advanced_config.html +++ b/app/templates/advanced_config.html @@ -209,6 +209,28 @@
Cloud Provider:
}); } + function selectedSiteImageChanged() { + // disable the other select of AppDB images + var selected = $("#selectedSiteImage option:selected" ).text(); + if (selected == " - Select one image - ") { + $("#selectedImage").prop('disabled', false); + } else { + $("#selectedImage").prop('disabled', true); + } + activateSubmit() + } + + function selectedImageChanged() { + // disable the other select of site images + var selected = $("#selectedImage option:selected" ).text(); + if (selected == " - Select one image - ") { + $("#selectedSiteImage").prop('disabled', false); + } else { + $("#selectedSiteImage").prop('disabled', true); + } + activateSubmit() + } + function activateSubmit() { $(".submitBtn").attr("disabled", false); } @@ -252,7 +274,7 @@
Cloud Provider:

-
@@ -260,7 +282,7 @@
Cloud Provider:
From 537ab87b10962701b9be3b5b9e730ee17a45c121 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 11:23:02 +0200 Subject: [PATCH 17/20] Fix error #550 --- app/templates/advanced_config.html | 1 + 1 file changed, 1 insertion(+) diff --git a/app/templates/advanced_config.html b/app/templates/advanced_config.html index 2efda5f7a..ff8febc30 100644 --- a/app/templates/advanced_config.html +++ b/app/templates/advanced_config.html @@ -130,6 +130,7 @@
Cloud Provider:
$('#siteImages').hide(); $('#fedcloudUsage').hide(); } + $("#selectedImage").prop('disabled', false); } function loadImages() { From 1315fb1c21eec9352ffe6998ed1b920b41a64821 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 11:44:12 +0200 Subject: [PATCH 18/20] Fix error #550 --- app/templates/advanced_config.html | 1 + 1 file changed, 1 insertion(+) diff --git a/app/templates/advanced_config.html b/app/templates/advanced_config.html index ff8febc30..10d5375ed 100644 --- a/app/templates/advanced_config.html +++ b/app/templates/advanced_config.html @@ -131,6 +131,7 @@
Cloud Provider:
$('#fedcloudUsage').hide(); } $("#selectedImage").prop('disabled', false); + $("#selectedSiteImage").prop('disabled', false); } function loadImages() { From 59c9b01b7fe0bfd60a884bba984a026270c361c5 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 13:41:20 +0200 Subject: [PATCH 19/20] Fix error #550 --- app/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 8bd680a02..1890b4438 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1004,9 +1004,9 @@ def createdep(): if "public" in site["networks"][vo]: pub_network_id = site["networks"][vo]["public"] - if form_data['extra_opts.selectedImage'] != "" and 'name' in site: + if form_data.get('extra_opts.selectedImage') not in ["", None] and 'name' in site: image = "appdb://%s/%s?%s" % (site['name'], form_data['extra_opts.selectedImage'], vo) - elif form_data['extra_opts.selectedSiteImage'] != "": + elif form_data.get('extra_opts.selectedSiteImage') not in ["", None]: image = form_data['extra_opts.selectedSiteImage'] else: image_id = form_data['extra_opts.imageID'] From 1ef468efb3037ce0797f7edc7b7665d56b7d1a31 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 16 May 2024 13:42:32 +0200 Subject: [PATCH 20/20] Fix error #550 --- app/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 1890b4438..dfd525547 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1004,9 +1004,9 @@ def createdep(): if "public" in site["networks"][vo]: pub_network_id = site["networks"][vo]["public"] - if form_data.get('extra_opts.selectedImage') not in ["", None] and 'name' in site: + if form_data.get('extra_opts.selectedImage', "") != "" and 'name' in site: image = "appdb://%s/%s?%s" % (site['name'], form_data['extra_opts.selectedImage'], vo) - elif form_data.get('extra_opts.selectedSiteImage') not in ["", None]: + elif form_data.get('extra_opts.selectedSiteImage', "") != "": image = form_data['extra_opts.selectedSiteImage'] else: image_id = form_data['extra_opts.imageID']