From 178375eeae18430dddccfd7e6c58566e7872c011 Mon Sep 17 00:00:00 2001 From: jsjiang Date: Thu, 15 Aug 2024 17:14:30 -0700 Subject: [PATCH 1/7] remove binder from the queuing services --- ezidapp/management/commands/proc-cleanup-async-queues.py | 2 -- impl/enqueue.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ezidapp/management/commands/proc-cleanup-async-queues.py b/ezidapp/management/commands/proc-cleanup-async-queues.py index 88a8bf2b..d9f578ff 100644 --- a/ezidapp/management/commands/proc-cleanup-async-queues.py +++ b/ezidapp/management/commands/proc-cleanup-async-queues.py @@ -35,7 +35,6 @@ class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand): setting = 'DAEMONS_QUEUE_CLEANUP_ENABLED' queueType = { - 'binder': ezidapp.models.async_queue.BinderQueue, 'crossref': ezidapp.models.async_queue.CrossrefQueue, 'datacite': ezidapp.models.async_queue.DataciteQueue, 'search': ezidapp.models.async_queue.SearchIndexerQueue @@ -72,7 +71,6 @@ def run(self): # set status for each handle system identifierStatus = { - 'binder' : False, 'crossref' : False, 'datacite' : False, 'search' : False diff --git a/impl/enqueue.py b/impl/enqueue.py index 82690f3b..aacf5ae2 100644 --- a/impl/enqueue.py +++ b/impl/enqueue.py @@ -49,7 +49,7 @@ def enqueue( ref_id_model = create_ref_id_model(si_model) queue_model_list = [ezidapp.models.async_queue.SearchIndexerQueue] - # Do not add reserved identifiers to binder, crossref, or datacite queues + # Do not add reserved identifiers to crossref, or datacite queues # When the identifier entry is updated to not be reserved, then the various # external services will be called as appropriate. # See https://github.com/CDLUC3/ezid/blob/v2.0.6/impl/backproc.py#L117 for @@ -58,7 +58,6 @@ def enqueue( if updateExternalServices: queue_model_list.extend( ( - ezidapp.models.async_queue.BinderQueue, ezidapp.models.async_queue.CrossrefQueue, ezidapp.models.async_queue.DataciteQueue, ) From 93df070e185e2df3532c4e8099f9bf29b83fbcfa Mon Sep 17 00:00:00 2001 From: jsjiang Date: Thu, 15 Aug 2024 17:18:00 -0700 Subject: [PATCH 2/7] remove binder from diag-identifier --- .../management/commands/diag-identifier.py | 158 ------------------ 1 file changed, 158 deletions(-) diff --git a/ezidapp/management/commands/diag-identifier.py b/ezidapp/management/commands/diag-identifier.py index 9dbc07c6..5d379fab 100644 --- a/ezidapp/management/commands/diag-identifier.py +++ b/ezidapp/management/commands/diag-identifier.py @@ -199,28 +199,6 @@ def add_arguments(self, parser:argparse.ArgumentParser): help="Ending date for metrics" ) - _syncmeta = subparsers.add_parser( - "syncmeta", - help=("Sends metadata to N2T for each row text file source. Rows starting with space or '#' are ignored.\n" - "Example:\n" - " ./manage.py diag-identifier syncmeta -f pid_list.txt" - ) - ) - _syncmeta.add_argument( - '-f', - '--from', - type=str, - help="Text file with one identifier per line.", - required=True - ) - _syncmeta.add_argument( - '-s', - '--start', - type=str, - help="Identifier in list to start from", - default=None - ) - def diff_n2t(self, identifier:ezidapp.models.identifier)->dict: res = {} @@ -242,52 +220,6 @@ def diff_n2t(self, identifier:ezidapp.models.identifier)->dict: return res - def prepare_n2t_metadata(self, identifier:ezidapp.models.identifier, n2t_meta:typing.Optional[dict]=None)->dict: - '''Prepare metadata for sending to N2T - - Returns a dictionary of metadata for identifier that can be sent to - N2T using impl.noid_egg.setElements(identifier.identifier, m) to - set or update the N2T entry for identifier. - - Metadata is sent to N2T for all states except Reserved DOIs, for which N2T is generally null. - ''' - _legacy = identifier.toLegacy() - # See proc_binder.update - if n2t_meta is None: - # Retrieve the existing metadata from N2T - n2t_meta = impl.noid_egg.getElements(identifier.identifier) - # if no metadata on N2T then initialize a blank for population. - if n2t_meta is None: - n2t_meta = {} - - if identifier.isReserved: - #special case - reserved - do nothing - log.info("Reserved DOIs have null N2T metadata.") - # To delete metadata on N2T, send keys with empty values, but we don't want to - # delete all the keys since that has the effect of deleting the identifier from N2T. - #for k in n2t_meta: - # n2t_meta[k] = "" - #return n2t_meta - return {} - - # First, update m with provided metadata - for k, v in list(_legacy.items()): - # If the provided metadata matches existing, then ignore - if n2t_meta.get(k) == v: - del n2t_meta[k] - # Otherwise add property to list for sending back to N2T - else: - n2t_meta[k] = v - # If properties retrieved from N2T are not present in the supplied - # update metadata, then set the value of the field to an empty string. - # An empty value results in an "rm" (remove) operation for that field - # being sent to N2T. - for k in list(n2t_meta.keys()): - if k not in _legacy: - n2t_meta[k] = "" - return n2t_meta - - def handle_show(self, *args, **opts): def jsonable_instance(o): if o is None: @@ -340,31 +272,6 @@ def tstamp_to_text(t): entry["cm_eq_metadata"] = _mequal except zlib.error: log.info("No cm section in %s", identifier.identifier) - n2t_meta = None - if opts["N2T"]: - # Retrieve entry from N2T - n2t_meta = impl.noid_egg.getElements(identifier.identifier) - entry["n2t"] = n2t_meta - if opts["sync"]: - _legacy = identifier.toLegacy() - # See proc_binder.update - # Retrieve the existing metadata from N2T - m = self.prepare_n2t_metadata(identifier, n2t_meta) - if len(m) > 0: - log.warning("Updating N2T metadata for %s", identifier.identifier) - log.info("Pending updates for %s:\n%s", identifier.identifier, m) - self.stdout.write(f"About to update {identifier.identifier} !") - response = input("Enter Y to continue, anything else aborts: ") - if response.strip() == 'Y': - impl.noid_egg.setElements(identifier.identifier, m) - ## - # Retrieve the updated metadata and add to the entry - entry["n2t_updated"] = impl.noid_egg.getElements(identifier.identifier) - else: - self.stdout.write("Aborted.") - else: - log.info("No pending updates for %s", identifier.identifier) - entries.append(entry) self.stdout.write(json.dumps(entries, indent=2, sort_keys=True)) @@ -476,69 +383,6 @@ def handle_metrics(self, *args, **opts): for row in cursor.fetchall(): writer.writerow(row) - - def handle_syncmeta(self, *args, **opts): - '''For each line in `from`: - update metadata - ''' - fn_src = opts.get('from') - fn_dst = fn_src + ".json" - start_at = opts.get("start", None) - log.info("Recording changes to %s", fn_dst) - identifiers = [] - add_id = True - if start_at is not None: - add_id = False - start_at = start_at.strip() - with open(fn_src) as _src: - for row in _src: - if row.startswith('ark:') or row.startswith('doi:'): - if not add_id: - if row.strip() == start_at: - add_id = True - if add_id: - identifiers.append(row.strip()) - log.info("Loaded %s identifiers from %s", len(identifiers), fn_src) - log.info("Loading status...") - with open(fn_dst, 'a') as f_dest: - for pid in identifiers: - self.stdout.write(pid) - result = {'pid':pid, 'original': {}, 'change': {}, 'updated':{}} - identifier = ezidapp.models.identifier.SearchIdentifier.objects.get(identifier=pid) - if identifier is None: - log.error('Identifier %s could not be loaded!', pid) - break - if identifier.isDatacite: - # handle datacite target url - doi = identifier.identifier[4:] - datacenter = str(identifier.datacenter) - log.info("Setting target for %s (%s) to %s", doi, datacenter, identifier.resolverTarget) - r = impl.datacite.setTargetUrl(doi, identifier.resolverTarget, datacenter) - if r is not None: - # There was a failure in the request - log.error("Failed to set target url for DataCite DOI: %s", doi) - pass - elif identifier.isCrossref: - # handle crossref target url - pass - result['original'] = impl.noid_egg.getElements(identifier.identifier) - n2t_meta = copy.deepcopy(result['original']) - result['change'] = self.prepare_n2t_metadata(identifier, n2t_meta=n2t_meta) - self.stdout.write(json.dumps(result['change'])) - if result['change'] != {}: - # Send update request - impl.noid_egg.setElements(identifier.identifier, result['change']) - # Retrieve the updated n2t meta - result['updated'] = impl.noid_egg.getElements(identifier.identifier) - else: - # no change - result['updated'] = result['original'] - f_dest.write(json.dumps(result)) - f_dest.write("\n") - f_dest.flush() - - - def handle(self, *args, **opts): operation = opts['operation'] if operation == 'show': @@ -552,7 +396,5 @@ def handle(self, *args, **opts): self.handle_resolve(*args, **opts) elif operation == 'metrics': self.handle_metrics(*args, **opts) - elif operation =='syncmeta': - self.handle_syncmeta(*args, **opts) From 2f1fcccbb34132a32fc4b8f589960381743bd0fd Mon Sep 17 00:00:00 2001 From: jsjiang Date: Thu, 15 Aug 2024 17:18:20 -0700 Subject: [PATCH 3/7] remove binder configs --- ansible/group_vars/all | 3 --- settings/settings.py.j2 | 7 ------- settings/tests.py | 7 ------- tests/util/create_settings.py | 3 --- tests/util/mk_test_settings.py | 3 --- 5 files changed, 23 deletions(-) diff --git a/ansible/group_vars/all b/ansible/group_vars/all index cb00737d..70096262 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -83,9 +83,6 @@ admin_username: "{{ ssm_params['admin_username'] }}" allocator_cdl_password: "{{ ssm_params['allocator_cdl_password'] }}" allocator_purdue_password: "{{ ssm_params['allocator_purdue_password'] }}" -binder_url: "{{ ssm_params['binder_url'] }}" -binder_username: "{{ ssm_params['binder_username'] }}" -binder_password: "{{ ssm_params['binder_password'] }}" cloudwatch_instance_name: "{{ ansible_facts.hostname }}" crossref_username: "{{ ssm_params['crossref_username'] }}" crossref_password: "{{ ssm_params['crossref_password'] }}" diff --git a/settings/settings.py.j2 b/settings/settings.py.j2 index 6d0fc206..9b63e92a 100644 --- a/settings/settings.py.j2 +++ b/settings/settings.py.j2 @@ -126,7 +126,6 @@ if DAEMONS_ENABLED == 'auto': # - True: The daemon is available to be started. # - False: The daemon cannot run. # - See the DAEMONS_ENABLED setting above. -DAEMONS_BINDER_ENABLED = True DAEMONS_QUEUE_CLEANUP_ENABLED = True DAEMONS_CROSSREF_ENABLED = True DAEMONS_DATACITE_ENABLED = True @@ -430,12 +429,6 @@ S3_BUCKET_DOWNLOAD_PATH = 'download' GZIP_COMMAND = '/usr/bin/gzip' ZIP_COMMAND = '/usr/bin/zip' -BINDER_URL = '{{ binder_url }}' -BINDER_USERNAME = '{{ binder_username }}' -BINDER_PASSWORD = '{{ binder_password }}' -BINDER_NUM_ATTEMPTS = 3 -BINDER_REATTEMPT_DELAY = 5 - # The ARK resolvers correspond to the above binders. RESOLVER_DOI = '{{ resolver_doi }}' RESOLVER_ARK = '{{ resolver_ark }}' diff --git a/settings/tests.py b/settings/tests.py index 2bf52493..8df2992e 100644 --- a/settings/tests.py +++ b/settings/tests.py @@ -128,7 +128,6 @@ # - True: The daemon is available to be started. # - False: The daemon cannot run. # - See the DAEMONS_ENABLED setting above. -DAEMONS_BINDER_ENABLED = True DAEMONS_CROSSREF_ENABLED = True DAEMONS_DATACITE_ENABLED = True DAEMONS_DOWNLOAD_ENABLED = True @@ -389,12 +388,6 @@ GZIP_COMMAND = '/usr/bin/gzip' ZIP_COMMAND = '/usr/bin/zip' -BINDER_URL = 'https://n2t-stg.n2t.net/a/ezid/b' -BINDER_USERNAME = 'ezid' -BINDER_PASSWORD = '' -BINDER_NUM_ATTEMPTS = 3 -BINDER_REATTEMPT_DELAY = 5 - # The ARK resolvers correspond to the above binders. RESOLVER_DOI = 'https://doi.org' RESOLVER_ARK = 'https://n2t-stg.n2t.net' diff --git a/tests/util/create_settings.py b/tests/util/create_settings.py index ceeacdf9..672d1846 100755 --- a/tests/util/create_settings.py +++ b/tests/util/create_settings.py @@ -23,9 +23,6 @@ 'email_new_account': 'invalid@invalid.invalid', 'admin_username': 'admin', 'admin_password': 'admin', - 'binder_url': 'https://n2t-stg.n2t.net/a/ezid/b', - 'binder_username': 'ezid', - 'binder_password': '', 'resolver_doi': 'https://doi.org', 'resolver_ark': 'https://n2t-stg.n2t.net', 'datacite_doi_url': 'https://mds.datacite.org/doi', diff --git a/tests/util/mk_test_settings.py b/tests/util/mk_test_settings.py index be57c5b0..e05f1aae 100755 --- a/tests/util/mk_test_settings.py +++ b/tests/util/mk_test_settings.py @@ -51,9 +51,6 @@ 'admin_search_user_pid': 'ark:/99166/p9kw57h4w', 'admin_search_group_pid': 'ark:/99166/p9g44hq02', # Misc - 'binder_url': 'https://n2t-stg.n2t.net/a/ezid/b', - 'binder_username': 'ezid', - 'binder_password': '', 'resolver_doi': 'https://doi.org', 'resolver_ark': 'https://n2t-stg.n2t.net', 'datacite_doi_url': 'https://mds.datacite.org/doi', From 8e2be2cb91b06d1717ef682ced4c0262fb64f821 Mon Sep 17 00:00:00 2001 From: jsjiang Date: Fri, 16 Aug 2024 12:05:50 -0700 Subject: [PATCH 4/7] Remove n2t/binder related code Remove noid_egg library and dependencies Remove proc-binder job Remove n2t diag-identifier tool --- .../management/commands/diag-identifier.py | 55 +--- ezidapp/management/commands/proc-binder.py | 119 --------- impl/api.py | 5 +- impl/noid_egg.py | 246 ------------------ impl/noid_nog_standalone.py | 50 ---- migrate_db_to_py3/diag-create-fixtures.py | 1 - 6 files changed, 2 insertions(+), 474 deletions(-) delete mode 100644 ezidapp/management/commands/proc-binder.py delete mode 100644 impl/noid_egg.py delete mode 100644 impl/noid_nog_standalone.py diff --git a/ezidapp/management/commands/diag-identifier.py b/ezidapp/management/commands/diag-identifier.py index 5d379fab..530f0d65 100644 --- a/ezidapp/management/commands/diag-identifier.py +++ b/ezidapp/management/commands/diag-identifier.py @@ -9,8 +9,6 @@ This command does not alter any information in the database, and should be safe to run at any time, including a running production instance. -Note however, that this command MAY alter the information in N2T when the --sync option -is used. Confirmation is requested before any metadata updates are propagated to N2T. """ import argparse @@ -36,7 +34,6 @@ import ezidapp.models.identifier import ezidapp.models.user import impl.datacite -import impl.noid_egg log = logging.getLogger(__name__) @@ -67,7 +64,7 @@ def add_arguments(self, parser:argparse.ArgumentParser): _show = subparsers.add_parser( "show", - help=("Show available metadata for an identifier, and optionally sync the N2T record.\n" + help=("Show available metadata for an identifier.\n" "Example:\n" " Default:\n" " ./manage.py diag-identifier show ark:/62930/d1n739\n") @@ -84,12 +81,6 @@ def add_arguments(self, parser:argparse.ArgumentParser): action='store_true', help='Show Identifier instead of SearchIdentifier table entry', ) - _show.add_argument( - '-y', - '--legacy', - action='store_true', - help='Show legacy form of identifier record', - ) _show.add_argument( '-m', '--cm', @@ -108,17 +99,6 @@ def add_arguments(self, parser:argparse.ArgumentParser): action='store_true', help='Convert timestamps to textual time representation', ) - _show.add_argument( - '-N', - '--N2T', - action='store_true', - help='Retrieve record from N2T if available', - ) - _show.add_argument( - '--sync', - action='store_true', - help="Synchronize the N2T entry with metadata from the database.", - ) _list = subparsers.add_parser( "list", @@ -154,11 +134,6 @@ def add_arguments(self, parser:argparse.ArgumentParser): default=[], help="Comma separated list of fields in addition to identifier to list." ) - _list.add_argument( - '--compare', - action='store_true', - help='Show difference between EZID and N2T metadata.', - ) _list.add_argument( '-m', '--max_rows', @@ -199,27 +174,6 @@ def add_arguments(self, parser:argparse.ArgumentParser): help="Ending date for metrics" ) - - def diff_n2t(self, identifier:ezidapp.models.identifier)->dict: - res = {} - n2t_meta = impl.noid_egg.getElements(identifier.identifier) - if n2t_meta is None: - n2t_meta = {} - _legacy = identifier.toLegacy() - for k, v in _legacy.items(): - res[k] = [v, None] - # If properties retrieved from N2T are not present in the supplied - # update metadata, then set the value of the field to an empty string. - # An empty value results in an "rm" (remove) operation for that field - # being sent to N2T. - for k, v in n2t_meta.items(): - if k not in res: - res[k] = [None, v] - else: - res[k][1] = v - return res - - def handle_show(self, *args, **opts): def jsonable_instance(o): if o is None: @@ -250,9 +204,6 @@ def tstamp_to_text(t): # but we want to futz around with the cm section and other fields for each instance. entry = jsonable_instance(identifier) entry["isAgentPid"] = identifier.isAgentPid - if opts["legacy"]: - # Get the "legacy" format, which is used for sending to N2T binder - entry["legacy"] = identifier.toLegacy() if opts["expanded"]: for field_name in expand_fields: entry["fields"][field_name] = jsonable_instance(getattr(identifier, field_name)) @@ -329,14 +280,10 @@ def handle_list(self, *args, **opts): identifier_class = ezidapp.models.identifier.Identifier identifiers = identifier_class.objects.filter(**_filter).order_by("-createTime")[:max_rows] dfields = _fields - if opts.get("compare", False): - dfields.append('n2t') writer = csv.DictWriter(self.stdout, dfields, dialect='excel') writer.writeheader() for identifier in identifiers: row = django.forms.models.model_to_dict(identifier, fields=_fields) - if opts.get('compare', False): - row['n2t'] = self.diff_n2t(identifier) writer.writerow(row) diff --git a/ezidapp/management/commands/proc-binder.py b/ezidapp/management/commands/proc-binder.py deleted file mode 100644 index c835a863..00000000 --- a/ezidapp/management/commands/proc-binder.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright©2021, Regents of the University of California -# http://creativecommons.org/licenses/BSD - -"""Asynchronous N2T binder processing -""" - -import logging - -import ezidapp.management.commands.proc_base -import ezidapp.models.async_queue -import ezidapp.models.identifier -import impl.log -import impl.noid_egg - -log = logging.getLogger(__name__) - - -class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand): - help = __doc__ - name = __name__ - setting = 'DAEMONS_BINDER_ENABLED' - queue = ezidapp.models.async_queue.BinderQueue - - def create(self, task_model: ezidapp.models.async_queue.BinderQueue): - """ - Creates an entry in N2T for a new identifier. - The fields to be set are described in the N2T API documentation: - http://n2t.net/e/n2t_apidoc.html - Minimally, the fields must include: - who - what - when - where - how Where is this value stored in EZID? - _t - """ - id_str = task_model.refIdentifier.identifier - self.log.info("CREATE: %s", id_str) - ##metadata = task_model.refIdentifier.metadata - # add the required target metadata: - ##metadata["_t"] = task_model.refIdentifier.target - metadata = task_model.refIdentifier.toLegacy() - try: - impl.noid_egg.setElements(id_str, metadata) - task_model.status = self.queue.SUCCESS - except AssertionError as e: - task_model.status = self.queue.FAILURE - self.log.error("CREATE: %s", id_str, e) - except Exception as e: - task_model.status = self.queue.FAILURE - self.log.error("CREATE: %s", id_str, e) - task_model.save() - - def update(self, task_model: ezidapp.models.async_queue.BinderQueue): - ''' - task_model: BinderQueue - - Retrieves existing metadata from N2T and sends back updates to any - new fields oor fields that have changed values. - ''' - id_str = task_model.refIdentifier.identifier - ##metadata = task_model.refIdentifier.metadata - ### add the required target metadata: - ##metadata["_t"] = task_model.refIdentifier.target - metadata = task_model.refIdentifier.toLegacy() - self.log.info("UPDATE: %s", id_str) - - # Retrieve the existing metadata from N2T - m = impl.noid_egg.getElements(id_str) - if m is None: - m = {} - # First, update m with provided metadata - for k, v in list(metadata.items()): - # If the provided metadata matches existing, then ignore - if m.get(k) == v: - del m[k] - # Otherwise add property to list for sending back to N2T - else: - m[k] = v - # If properties retrieved from N2T are not present in the supplied - # update metadata, then set the value of the field to an empty string. - # An empty value results in an "rm" (remove) operation for that field - # being sent to N2T. - for k in list(m.keys()): - if k not in metadata: - m[k] = "" - self.log.debug("UPDATE: %s m = %s", id_str, m) - if len(m) > 0: - try: - impl.noid_egg.setElements(id_str, m) - task_model.status = self.queue.SUCCESS - except AssertionError as e: - task_model.status = self.queue.FAILURE - self.log.error("UPDATE: %s", id_str, e) - except Exception as e: - task_model.status = self.queue.FAILURE - self.log.error("UPDATE: %s", id_str, e) - task_model.save() - - def delete(self, task_model: ezidapp.models.async_queue.BinderQueue): - id_str = task_model.refIdentifier.identifier - try: - impl.noid_egg.deleteIdentifier(id_str) - task_model.status = self.queue.SUCCESS - except AssertionError as e: - task_model.status = self.queue.FAILURE - self.log.error("DELETE: %s", id_str, e) - except Exception as e: - task_model.status = self.queue.FAILURE - self.log.error("DELETE: %s", id_str, e) - task_model.save() - - def batchCreate(self, batch): - impl.noid_egg.batchSetElements(batch) - - def batchDelete(self, batch): - impl.noid_egg.batchDeleteIdentifier( - [identifier for identifier, metadata in batch], - ) diff --git a/impl/api.py b/impl/api.py index 444c28a9..6314aa65 100644 --- a/impl/api.py +++ b/impl/api.py @@ -102,7 +102,6 @@ import impl.datacite import impl.download import impl.ezid -import impl.noid_egg import impl.resolver import impl.search_util import impl.statistics @@ -427,9 +426,7 @@ def getStatus(request): if l == "*": l = "binder,datacite,search" for ss in [ss.strip() for ss in l.split(",") if len(ss.strip()) > 0]: - if ss == "binder": - body += f"binder: {impl.noid_egg.ping()}\n" - elif ss == "datacite": + if ss == "datacite": body += f"datacite: {impl.datacite.ping()}\n" elif ss == "search": body += f"search: {impl.search_util.ping()}\n" diff --git a/impl/noid_egg.py b/impl/noid_egg.py deleted file mode 100644 index faec2188..00000000 --- a/impl/noid_egg.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright©2021, Regents of the University of California -# http://creativecommons.org/licenses/BSD - -"""Interface to the "egg" (binder) portion of noid - -A note on encodings. Identifiers and metadata elements (both names and values) are sent -to noid in encoded form; see util.encode{3,4}. Metadata elements received from void are -UTF-8-encoded and utilize percent-encoding. Though this received encoding does not -exactly match the transmitted encoding, the decoding performed by util.decode is -nevertheless compatible and so we use it. (Consider a Python Unicode value -u"Greg%Jan\xe9e". This is sent as "Greg%25Jan%C3%A9e" but received back as -"Greg%25Jan\xc3\xa9e", which, when percent- and UTF-8-decoded, yields the original -value.) - -This module performs whitespace processing. Leading and trailing whitespace is stripped -from both element names and values. Empty names are not allowed. Setting an empty -value causes the element to be deleted; as a consequence, empty values are never -returned. -""" - -import logging -import re -import time -import urllib.error -import urllib.parse -import urllib.request -import urllib.response - -import django.conf - -import impl.log -import impl.util - -log = logging.getLogger(__name__) - -DECODE_RX = re.compile("\^([0-9a-fA-F][0-9a-fA-F])?") - - -@impl.log.stacklog -def _issue(method, operations): - # noinspection PyUnresolvedReferences - r = urllib.request.Request(django.conf.settings.BINDER_URL + "?-") - r.get_method = lambda: method - # noinspection PyTypeChecker - r.add_header( - "Authorization", - impl.util.basic_auth( - django.conf.settings.BINDER_USERNAME, - django.conf.settings.BINDER_PASSWORD, - ), - ) - _timeout = 60 #seconds - try: - _timeout = django.conf.settings.DAEMONS_HTTP_CLIENT_TIMEOUT - except AttributeError: - log.warning("No settings.DAEMONS_HTTP_CLIENT_TIMEOUT. Using default of %s", _timeout) - - r.add_header("Content-Type", "text/plain") - - s = "" - - l = [] - for o in operations: - # o = (identifier, operation [,element [, value]]) - s = f":hx% {impl.util.encode4(o[0])}.{o[1]}" - if len(o) > 2: - s += " " + impl.util.encode4(o[2]) - if len(o) > 3: - s += " " + impl.util.encode3(o[3]) - l.append(s) - r.data = "\n".join(l).encode('utf-8') - - for i in range(django.conf.settings.BINDER_NUM_ATTEMPTS): - c = None - log.debug("noid_egg._issue attempt:%s %s url:%s", i, method, r.full_url) - try: - c = urllib.request.urlopen(r, timeout=_timeout) - s = [line.decode('utf-8', errors='replace') for line in c.readlines()] - except Exception as e: - # noinspection PyTypeChecker - log.warning("noid_egg._issue attempt:%s exception: %s", i, e) - if i == django.conf.settings.BINDER_NUM_ATTEMPTS - 1: - raise - else: - break - finally: - if c: - c.close() - # noinspection PyTypeChecker - # increase reattempt delay as a magnitude of BINDER_NUM_ATTEMPTS - time.sleep(django.conf.settings.BINDER_REATTEMPT_DELAY + (60 * (i + 1))) - - return s - - -def _error(operation, s): - return f'unexpected return from noid egg "{operation}":\n ' f'{"".join(str(x) for x in s)}' - - -def identifierExists(id_str): - """Return true if an identifier (given in normalized, qualified form, - e.g., "doi:10.1234/FOO") exists. - - Raises an exception on error. - """ - # The question of whether an identifier exists or not is surprisingly elusive. Noid will return - # information for any identifier string, so we can't use that as a test. Instead, we test for - # the presence of metadata. EZID populates a newly-created identifier with multiple metadata - # fields. (Noid adds its own internal metadata fields, but only in response to EZID adding - # fields.) - # - # The 'getElements' and 'deleteIdentifier' functions below work to maintain the invariant - # property that either an identifier has EZID metadata (along with noid-internal metadata) or it - # has no metadata at all. - s = _issue("GET", [(id_str, "fetch")]) - assert ( - len(s) >= 4 - and s[0].startswith("# id:") - and s[-3].startswith("# elements bound under") - and s[-2] == "egg-status: 0\n" - ), _error("fetch", s) - m = re.search(": (\\d+)\n$", s[-3]) - assert m, _error("fetch", s) - return m.group(1) != "0" - - -def setElements(id_str, d): - """Bind metadata elements to an id_str (given in normalized, qualified - form, e.g., "doi:10.1234/FOO"). - - The elements should be given in a dictionary that maps names to - values. Raises an exception on error. - - Setable elements are described in the N2T API docs: http://n2t.net/e/n2t_apidoc.html - """ - batchSetElements([(id_str, d)]) - - -def batchSetElements(batch): - """Similar to 'setElements' above, but operates on multiple identifiers in - one request. - - 'batch' should be a list of (identifier, name/value dictionary) - tuples. - """ - bind_list = [] - for identifier, d in batch: - for e, v in list(d.items()): - e = e.strip() - assert len(e) > 0, "empty label" - v = v.strip() - if v == "": - bind_list.append((identifier, "rm", e)) - else: - bind_list.append((identifier, "set", e, v)) - s = _issue("POST", bind_list) - assert len(s) >= 2 and s[-2] == "egg-status: 0\n", _error("set/rm", s) - - -def getElements(identifier): - """Return all metadata elements (in the form of a dictionary) that are - bound to an identifier (given in normalized, qualified form, e.g., - "doi:10.1234/FOO"), or None if the identifier doesn't exist. - - Raises an exception on error. - """ - # See the comment under 'identifierExists' above. - s = _issue("GET", [(identifier, "fetch")]) - assert ( - len(s) >= 4 - and s[0].startswith("# id:") - and s[-3].startswith("# elements bound under") - and s[-2] == "egg-status: 0\n" - ), _error("fetch", s) - m = re.search(": (\\d+)\n$", s[-3]) - assert m, _error("fetch", s) - c = int(m.group(1)) - assert len(s) == c + 4, _error("fetch", s) - if c == 0: - return None - else: - d = {} - for l in s[1 : len(s) - 3]: - assert ":" in l, _error("fetch", s) - if l.startswith("__") or l.startswith("_.e") or l.startswith("_,e"): - continue - e, v = l.split(":", 1) - d[impl.util.decode(e)] = impl.util.decode(v.strip()) - # There had better be at least one non-noid-internal binding. - assert len(d) > 0, _error("fetch", s) - return d - - -def deleteIdentifier(identifier): - """Delete all metadata elements (including noid-internal elements) bound - to an identifier (given in normalized, qualified form, e.g., - "doi:10.1234/FOO"). - - After calling this function, the identifier is deleted in the sense - that identifierExists(identifier) will return False and - getElements(identifier) will return None. As far as noid is - concerned, however, the identifier still exists and metadata - elements can be re-bound to it in the future. Raises an exception - on error. - """ - s = _issue("POST", [(identifier, "purge")]) - assert len(s) >= 2 and s[-2] == "egg-status: 0\n", _error("purge", s) - # See the comment under 'identifierExists' above. - assert not identifierExists( - identifier - ), f"noid egg 'purge' operation on {identifier} left remaining bindings" - - -def batchDeleteIdentifier(batch): - """Similar to 'deleteIdentifier' above, but deletes a list of identifiers - in one request.""" - # The following code does not verify that all bindings have been - # removed as 'deleteIdentifier' does above. But that code is just a - # guard against noid API changes, and having it in one place is - # sufficient. - s = _issue("POST", [(identifier, "purge") for identifier in batch]) - assert len(s) >= 2 and s[-2] == "egg-status: 0\n", _error("purge", s) - - -def ping(): - """Test the server, returning "up" or "down".""" - try: - s = _issue("GET", []) - assert len(s) >= 2 and s[-2] == "egg-status: 0\n" - return "up" - except Exception: - return "down" - - -def _decodeRewriter(m): - assert len(m.group(0)) == 3, "circumflex decode error" - return chr(int(m.group(0)[1:], 16)) - - -def decodeRaw(s): - """Decode an identifier or metadata element name as stored internally in - noid. - - Raises AssertionError and UnicodeDecodeError. - """ - return DECODE_RX.sub(_decodeRewriter, s) diff --git a/impl/noid_nog_standalone.py b/impl/noid_nog_standalone.py deleted file mode 100644 index 00759254..00000000 --- a/impl/noid_nog_standalone.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright©2021, Regents of the University of California -# http://creativecommons.org/licenses/BSD - -"""Standalone version of noid_nog.py for use by offline tools -""" - -import base64 -import urllib.error -import urllib.parse -import urllib.request -import urllib.response - - -class Minter(object): - """A minter for a specific shoulder.""" - - def __init__(self, url, username, password): - """Create an interface to the noid nog minter at the supplied URL - using the supplied credentials.""" - self.url = url - self.username = username - self.password = password - - def _addAuthorization(self, request): - request.add_header( - "Authorization", - b"Basic " + (base64.b64encode(self.username + ":" + self.password)), - ) - - def mintIdentifier(self): - """Mint and returns a scheme-less ARK identifier, e.g., - "13030/fk35717n0h". - - Raises an exception on error. - """ - r = urllib.request.Request(self.url + "?mint%201") - self._addAuthorization(r) - c = None - try: - c = urllib.request.urlopen(r) - s = c.readlines() - finally: - if c: - c.close() - assert ( - len(s) >= 2 - and (s[0].startswith("id:") or s[0].startswith("s:")) - and s[-2] == "nog-status: 0\n" - ), "unexpected return from minter, output follows\n" + "".join(s) - return s[0].split(":", 1)[1].strip() diff --git a/migrate_db_to_py3/diag-create-fixtures.py b/migrate_db_to_py3/diag-create-fixtures.py index ae4d0c10..deb15e00 100644 --- a/migrate_db_to_py3/diag-create-fixtures.py +++ b/migrate_db_to_py3/diag-create-fixtures.py @@ -25,7 +25,6 @@ import impl.nog_sql.filesystem import impl.nog_sql.shoulder import impl.nog_sql.util -import impl.noid_egg APP_LABEL = 'ezidapp' From 07c5c092afc2c7d9fbcd59f0856804500b10c927 Mon Sep 17 00:00:00 2001 From: jsjiang Date: Thu, 29 Aug 2024 17:15:36 -0700 Subject: [PATCH 5/7] remove binder queue related code --- ansible/test_vars.yaml | 2 -- ezidapp/management/commands/check-ezid.py | 1 - impl/api.py | 3 +-- impl/statistics.py | 5 ----- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/ansible/test_vars.yaml b/ansible/test_vars.yaml index 88414f11..5ec6ef08 100644 --- a/ansible/test_vars.yaml +++ b/ansible/test_vars.yaml @@ -37,8 +37,6 @@ - name: return a single param based on ssm_root_path as variable ##debug: msg="{{ ssm_params['database_host'] }}" debug: msg="database_host {{ database_host }}" - - name: return binder_url - debug: msg="{{ binder_url }}" - name: return resolver_ark debug: msg="{{ resolver_ark }}" diff --git a/ezidapp/management/commands/check-ezid.py b/ezidapp/management/commands/check-ezid.py index 6c69996f..e37ac06e 100644 --- a/ezidapp/management/commands/check-ezid.py +++ b/ezidapp/management/commands/check-ezid.py @@ -69,7 +69,6 @@ } queueType = { - 'binder': ezidapp.models.async_queue.BinderQueue, 'crossref': ezidapp.models.async_queue.CrossrefQueue, 'datacite': ezidapp.models.async_queue.DataciteQueue, 'search': ezidapp.models.async_queue.SearchIndexerQueue diff --git a/impl/api.py b/impl/api.py index 6314aa65..086180ea 100644 --- a/impl/api.py +++ b/impl/api.py @@ -424,7 +424,7 @@ def getStatus(request): if "subsystems" in options: l = options["subsystems"] if l == "*": - l = "binder,datacite,search" + l = "datacite,search" for ss in [ss.strip() for ss in l.split(",") if len(ss.strip()) > 0]: if ss == "datacite": body += f"datacite: {impl.datacite.ping()}\n" @@ -478,7 +478,6 @@ def _statusLineGenerator(includeSuccessLine): f"STATUS {'paused' if isPaused else 'running'} " f"activeOperations={sum(activeUsers.values())} " f"waitingRequests={sum(waitingUsers.values())} " - f"binderQueueLength={impl.statistics.getBinderQueueLength()} " f"dataciteQueueLength={impl.statistics.getDataCiteQueueLength()} " "\n" ) diff --git a/impl/statistics.py b/impl/statistics.py index 44eefb03..b0fd1e1c 100644 --- a/impl/statistics.py +++ b/impl/statistics.py @@ -7,11 +7,6 @@ import ezidapp.models.async_queue -def getBinderQueueLength(): - """Return the length of the binder queue.""" - return ezidapp.models.async_queue.BinderQueue.objects.count() - - def getDataCiteQueueLength(): """Return the length of the DataCite queue.""" return ezidapp.models.async_queue.DataciteQueue.objects.count() From 5cbe4ef26569dc5e02bd9bd0cf2ef7a820691b39 Mon Sep 17 00:00:00 2001 From: jsjiang Date: Tue, 3 Sep 2024 13:12:04 -0700 Subject: [PATCH 6/7] Refactor EZID client library Remove duplicated ezid client util scripts Rename misleading variables Add script to test ezid client --- tests/apicli_py2.py | 216 --------------------------------- tests/apicli_py3.py | 216 --------------------------------- tests/util/ezid_client.py | 6 +- tests/util/ezid_client_test.py | 42 +++++++ 4 files changed, 45 insertions(+), 435 deletions(-) delete mode 100644 tests/apicli_py2.py delete mode 100644 tests/apicli_py3.py create mode 100644 tests/util/ezid_client_test.py diff --git a/tests/apicli_py2.py b/tests/apicli_py2.py deleted file mode 100644 index 1f93200a..00000000 --- a/tests/apicli_py2.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright©2021, Regents of the University of California -# http://creativecommons.org/licenses/BSD - -"""Minimal EZID API client lib to support testing - -This minimal client lib for the EZID API is intended for supporting integrations tests and hence is -developed using the same version of python utilized by the EZID application. - -Based on https://github.com/CDLUC3/ezid-client-tools -""" - -import codecs -import logging -import re -import time -import urllib.error -import urllib.parse -import urllib.request -import urllib.response - - -class EZIDHTTPErrorProcessor(urllib.request.HTTPErrorProcessor): - def http_response(self, request, response): - # Bizarre that Python leaves this out. - if response.status == 201: - return response - else: - return urllib.request.HTTPErrorProcessor.http_response( - self, request, response - ) - - https_response = http_response - - -class EZIDClient(object): - def __init__( - self, - server_url, - session_id=None, - username=None, - password=None, - encoding="utf-8", - ): - self._L = logging.getLogger(self.__class__.__name__) - self._settings.BINDER_URL = server_url.strip("/") - self._cookie = session_id - self._encoding = encoding - self._opener = urllib.request.build_opener(EZIDHTTPErrorProcessor()) - self._username = username # preserve for test validation - if self._cookie is None: - self._setAuthHandler(username, password) - - def _encode(self, id_str): - return urllib.parse.quote(id_str, ":/") - - def _setAuthHandler(self, username, password): - h = urllib.request.HTTPBasicAuthHandler() - # noinspection PyUnresolvedReferences - h.add_password("EZID", self._settings.BINDER_URL, username, password) - self._opener.add_handler(h) - - def formatAnvlRequest(self, args): - request = [] - for i in range(0, len(args), 2): - k = args[i].decode(self._encoding) - if k == "@": - f = codecs.open(args[i + 1], encoding=self._encoding) - request += [l.strip("\r\n") for l in f.readlines()] - f.close() - else: - if k == "@@": - k = "@" - else: - k = re.sub("[%:\r\n]", lambda c: f"%{ord(c.group(0)):02X}", k) - v = args[i + 1].decode(self._encoding) - if v.startswith("@@"): - v = v[1:] - elif v.startswith("@") and len(v) > 1: - f = codecs.open(v[1:], encoding=self._encoding) - v = f.read() - f.close() - v = re.sub("[%\r\n]", lambda c: f"%{ord(c.group(0)):02X}", v) - request.append(f"{k}: {v}") - return "\n".join(request) - - def anvlresponseToDict( - self, response, format_timestamps=True, decode=False, _encoding="utf-8" - ): - res = {"status": "unknown", "status_message": "no content", "body": ""} - if response is None: - return res - response = response.splitlines() - # Treat the first response line as the status - K, V = response[0].split(":", 1) - res["status"] = K - res["status_message"] = V.strip(" ") - for line in response[1:]: - try: - K, V = line.split(":", 1) - V = V.strip() - if format_timestamps and (K == "_created:" or K == "_updated:"): - ls = line.split(":") - V = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(int(ls[1]))) - if decode: - V = re.sub( - "%([0-9a-fA-F][0-9a-fA-F])", - lambda m: chr(int(m.group(1), 16)), - V, - ) - self._L.debug("K : V = %s : %s", K, V) - res[K] = V - except ValueError: - res["body"] += line - return res - - def anvlResponseToText( - self, - response, - sort_lines=False, - format_timestamps=True, - decode=False, - one_line=False, - encoding="utf-8", - ): - lines = [] - if response is None: - return None - response = response.splitlines() - if sort_lines and len(response) >= 1: - statusLine = response[0] - response = response[1:] - response.sort() - response.insert(0, statusLine) - for line in response: - if format_timestamps and ( - line.startswith("_created:") or line.startswith("_updated:") - ): - ls = line.split(":") - line = ( - ls[0] - + ": " - + time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(int(ls[1]))) - ) - if decode: - line = re.sub( - "%([0-9a-fA-F][0-9a-fA-F])", - lambda m: chr(int(m.group(1), 16)), - line, - ) - if one_line: - line = line.replace("\n", " ").replace("\r", " ") - lines.append(line.encode(encoding)) - return b"\n".join(lines) - - def issueRequest(self, path, method, data=None, dest_f=None): - url = f"{self._settings.BINDER_URL}/{path}" - self._L.info("sending request: %s", url) - request = urllib.request.Request(url) - request.get_method = lambda: method - response = None - if data is not None: - request.add_header("Content-Type", "text/plain; charset=utf-8") - # noinspection PyUnresolvedReferences - request.data = data.encode("utf-8") - if self._cookie is not None: - request.add_header("Cookie", self._cookie) - try: - connection = self._opener.open(request) - if not dest_f is None: - while True: - dest_f.write(connection.read(1)) - dest_f.flush() - else: - response = connection.read() - return response.decode("utf-8"), connection.info() - except urllib.error.HTTPError as e: - self._L.error(f"{e.code:d} {str(e)}") - if e.fp is not None: - response = e.fp.read() - self._L.error(response) - return response, {} - - def login(self, username=None, password=None): - if not username is None: - self._setAuthHandler(username, password) - self._cookie = None - response, headers = self.issueRequest("login", "GET") - try: - self._cookie = headers.get("set-cookie", "").split(";")[0].split("=")[1] - response += f"\nsessionid={self._cookie}\n" - except IndexError: - self._L.warning("No sessionid cookie in response.") - return self.anvlresponseToDict(response) - - def logout(self): - response, headers = self.issueRequest("logout", "GET") - return self.anvlresponseToDict(response) - - def status(self): - response, headers = self.issueRequest("status", "GET") - return self.anvlresponseToDict(response) - - def mint(self, shoulder, params=None): - if params is None: - params = [] - data = self.formatAnvlRequest(params) - url = "shoulder/" + self._encode(shoulder) - response, headers = self.issueRequest(url, "POST", data=data) - return self.anvlresponseToDict(response) - - def view(self, pid, bang=False): - path = "id/" + self._encode(pid) - if bang: - path += "?prefix_match=yes" - response, headers = self.issueRequest(path, "GET") - return self.anvlresponseToDict(response) diff --git a/tests/apicli_py3.py b/tests/apicli_py3.py deleted file mode 100644 index 1f93200a..00000000 --- a/tests/apicli_py3.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright©2021, Regents of the University of California -# http://creativecommons.org/licenses/BSD - -"""Minimal EZID API client lib to support testing - -This minimal client lib for the EZID API is intended for supporting integrations tests and hence is -developed using the same version of python utilized by the EZID application. - -Based on https://github.com/CDLUC3/ezid-client-tools -""" - -import codecs -import logging -import re -import time -import urllib.error -import urllib.parse -import urllib.request -import urllib.response - - -class EZIDHTTPErrorProcessor(urllib.request.HTTPErrorProcessor): - def http_response(self, request, response): - # Bizarre that Python leaves this out. - if response.status == 201: - return response - else: - return urllib.request.HTTPErrorProcessor.http_response( - self, request, response - ) - - https_response = http_response - - -class EZIDClient(object): - def __init__( - self, - server_url, - session_id=None, - username=None, - password=None, - encoding="utf-8", - ): - self._L = logging.getLogger(self.__class__.__name__) - self._settings.BINDER_URL = server_url.strip("/") - self._cookie = session_id - self._encoding = encoding - self._opener = urllib.request.build_opener(EZIDHTTPErrorProcessor()) - self._username = username # preserve for test validation - if self._cookie is None: - self._setAuthHandler(username, password) - - def _encode(self, id_str): - return urllib.parse.quote(id_str, ":/") - - def _setAuthHandler(self, username, password): - h = urllib.request.HTTPBasicAuthHandler() - # noinspection PyUnresolvedReferences - h.add_password("EZID", self._settings.BINDER_URL, username, password) - self._opener.add_handler(h) - - def formatAnvlRequest(self, args): - request = [] - for i in range(0, len(args), 2): - k = args[i].decode(self._encoding) - if k == "@": - f = codecs.open(args[i + 1], encoding=self._encoding) - request += [l.strip("\r\n") for l in f.readlines()] - f.close() - else: - if k == "@@": - k = "@" - else: - k = re.sub("[%:\r\n]", lambda c: f"%{ord(c.group(0)):02X}", k) - v = args[i + 1].decode(self._encoding) - if v.startswith("@@"): - v = v[1:] - elif v.startswith("@") and len(v) > 1: - f = codecs.open(v[1:], encoding=self._encoding) - v = f.read() - f.close() - v = re.sub("[%\r\n]", lambda c: f"%{ord(c.group(0)):02X}", v) - request.append(f"{k}: {v}") - return "\n".join(request) - - def anvlresponseToDict( - self, response, format_timestamps=True, decode=False, _encoding="utf-8" - ): - res = {"status": "unknown", "status_message": "no content", "body": ""} - if response is None: - return res - response = response.splitlines() - # Treat the first response line as the status - K, V = response[0].split(":", 1) - res["status"] = K - res["status_message"] = V.strip(" ") - for line in response[1:]: - try: - K, V = line.split(":", 1) - V = V.strip() - if format_timestamps and (K == "_created:" or K == "_updated:"): - ls = line.split(":") - V = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(int(ls[1]))) - if decode: - V = re.sub( - "%([0-9a-fA-F][0-9a-fA-F])", - lambda m: chr(int(m.group(1), 16)), - V, - ) - self._L.debug("K : V = %s : %s", K, V) - res[K] = V - except ValueError: - res["body"] += line - return res - - def anvlResponseToText( - self, - response, - sort_lines=False, - format_timestamps=True, - decode=False, - one_line=False, - encoding="utf-8", - ): - lines = [] - if response is None: - return None - response = response.splitlines() - if sort_lines and len(response) >= 1: - statusLine = response[0] - response = response[1:] - response.sort() - response.insert(0, statusLine) - for line in response: - if format_timestamps and ( - line.startswith("_created:") or line.startswith("_updated:") - ): - ls = line.split(":") - line = ( - ls[0] - + ": " - + time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(int(ls[1]))) - ) - if decode: - line = re.sub( - "%([0-9a-fA-F][0-9a-fA-F])", - lambda m: chr(int(m.group(1), 16)), - line, - ) - if one_line: - line = line.replace("\n", " ").replace("\r", " ") - lines.append(line.encode(encoding)) - return b"\n".join(lines) - - def issueRequest(self, path, method, data=None, dest_f=None): - url = f"{self._settings.BINDER_URL}/{path}" - self._L.info("sending request: %s", url) - request = urllib.request.Request(url) - request.get_method = lambda: method - response = None - if data is not None: - request.add_header("Content-Type", "text/plain; charset=utf-8") - # noinspection PyUnresolvedReferences - request.data = data.encode("utf-8") - if self._cookie is not None: - request.add_header("Cookie", self._cookie) - try: - connection = self._opener.open(request) - if not dest_f is None: - while True: - dest_f.write(connection.read(1)) - dest_f.flush() - else: - response = connection.read() - return response.decode("utf-8"), connection.info() - except urllib.error.HTTPError as e: - self._L.error(f"{e.code:d} {str(e)}") - if e.fp is not None: - response = e.fp.read() - self._L.error(response) - return response, {} - - def login(self, username=None, password=None): - if not username is None: - self._setAuthHandler(username, password) - self._cookie = None - response, headers = self.issueRequest("login", "GET") - try: - self._cookie = headers.get("set-cookie", "").split(";")[0].split("=")[1] - response += f"\nsessionid={self._cookie}\n" - except IndexError: - self._L.warning("No sessionid cookie in response.") - return self.anvlresponseToDict(response) - - def logout(self): - response, headers = self.issueRequest("logout", "GET") - return self.anvlresponseToDict(response) - - def status(self): - response, headers = self.issueRequest("status", "GET") - return self.anvlresponseToDict(response) - - def mint(self, shoulder, params=None): - if params is None: - params = [] - data = self.formatAnvlRequest(params) - url = "shoulder/" + self._encode(shoulder) - response, headers = self.issueRequest(url, "POST", data=data) - return self.anvlresponseToDict(response) - - def view(self, pid, bang=False): - path = "id/" + self._encode(pid) - if bang: - path += "?prefix_match=yes" - response, headers = self.issueRequest(path, "GET") - return self.anvlresponseToDict(response) diff --git a/tests/util/ezid_client.py b/tests/util/ezid_client.py index 76c9c037..aba08c8a 100644 --- a/tests/util/ezid_client.py +++ b/tests/util/ezid_client.py @@ -43,7 +43,7 @@ def __init__( encoding="utf-8", ): self._L = logging.getLogger(self.__class__.__name__) - self._settings = django.conf.settings.BINDER_URL = server_url.strip("/") + self._SERVER_URL = server_url.strip("/") self._cookie = session_id self._encoding = encoding self._opener = urllib.request.build_opener(EZIDHTTPErrorProcessor()) @@ -56,7 +56,7 @@ def _encode(self, id_str): def _setAuthHandler(self, username, password): h = urllib.request.HTTPBasicAuthHandler() - h.add_password("EZID", self._settings.BINDER_URL, username, password) + h.add_password("EZID", self._SERVER_URL, username, password) self._opener.add_handler(h) def formatAnvlRequest(self, args): @@ -153,7 +153,7 @@ def anvlResponseToText( return b"\n".join(lines) def issueRequest(self, path, method, data=None, dest_f=None): - url = f"{self._settings.BINDER_URL}/{path}" + url = f"{self._SERVER_URL}/{path}" self._L.info("sending request: %s", url) request = urllib.request.Request(url) request.get_method = lambda: method diff --git a/tests/util/ezid_client_test.py b/tests/util/ezid_client_test.py new file mode 100644 index 00000000..0ef5a3f3 --- /dev/null +++ b/tests/util/ezid_client_test.py @@ -0,0 +1,42 @@ +from ezid_client import EZIDClient as ezid_client + +import argparse + +def main(): + # Create the parser + parser = argparse.ArgumentParser(description="Use the EZID API client library ezid_client.py to test EZID.") + + parser.add_argument('-e', '--env', type=str, required=True, choices=['test', 'dev', 'stg', 'prd'], help='Environment') + parser.add_argument('-u', '--username', type=str, required=False, help='user name') + parser.add_argument('-p', '--password', type=str, required=False, help='password') + + args = parser.parse_args() + env = args.env + username = args.username + password = args.password + + server_url = { + 'test': 'http://127.0.0.1:8000/', + 'dev': 'https://ezid-dev.cdlib.org/', + 'stg': 'https://ezid-stg.cdlib.org/', + 'prd': 'https://ezid.cdlib.org/' + } + + client = ezid_client(server_url.get(env), username=username, password=password) + + print("view") + ret = client.view("ark:/13030/m5z94194") + print(ret) + + print("mint") + ret = client.mint("ark:/99999/fk4") + print(ret) + + print("status") + ret = client.status() + print(ret) + +if __name__ == "__main__": + main() + + From 8f3cbcbc969ffcc8b9d9e36c70e134c132109ccc Mon Sep 17 00:00:00 2001 From: jsjiang Date: Tue, 3 Sep 2024 14:54:29 -0700 Subject: [PATCH 7/7] remove obsolete function call --- ezidapp/management/commands/diag-db-stats.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ezidapp/management/commands/diag-db-stats.py b/ezidapp/management/commands/diag-db-stats.py index 559c8df7..0e20adcd 100644 --- a/ezidapp/management/commands/diag-db-stats.py +++ b/ezidapp/management/commands/diag-db-stats.py @@ -72,14 +72,6 @@ def print_identifier(self, identifier): # print(id_model) # pprint.pp(id_model.cm) - print('-' * 100) - - impl.enqueue.enqueueBinderIdentifier( - identifier=id_model.identifier, - operation='update', - blob={'x': 'y'}, - ) - # impl.nog.util.print_table(row_list, log.info) def print_all(self):