From 99f50b5693da773ea647911958aa87bc64429348 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Wed, 24 Jul 2024 18:23:52 -0400 Subject: [PATCH 01/22] revert back to v1 zendesk action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57ed95bc4..29f8e0a0b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,7 @@ jobs: git push origin ${{ github.event.inputs.version }} - name: Create GitHub release (triggers publish-to-pypi workflow) - uses: zendesk/action-create-release@v2 + uses: zendesk/action-create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} with: From 17d16c5932b699a831e0ed24374fa67e47eba617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=20Ke=C3=AFn?= <3181182+alphasentaurii@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:54:32 -0400 Subject: [PATCH 02/22] core/allow-context-latest (#1062) * translate latest to operational context * incl state in server call to get_default_context * update changelog --- CHANGES.rst | 8 ++++++++ crds/client/api.py | 4 ++-- crds/core/config.py | 4 ++-- crds/core/heavy_client.py | 15 ++++++++++++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f9894cc1a..e69ed8dc7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +11.18.4 (unreleased) +==================== + +General +------- + +- Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] + 11.18.3 (2024-09-03) ==================== diff --git a/crds/client/api.py b/crds/client/api.py index 8d5e7c059..03cc652f8 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -308,11 +308,11 @@ def get_aui_best_references(date, dataset_ids): return bestrefs_map @utils.cached -def get_default_context(observatory=None): +def get_default_context(observatory=None, state="operational"): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ - return str(S.get_default_context(observatory)) + return str(S.get_default_context(observatory, state)) @utils.cached def get_context_by_date(date, observatory=None): diff --git a/crds/core/config.py b/crds/core/config.py index e83fd72a9..4e6169711 100644 --- a/crds/core/config.py +++ b/crds/core/config.py @@ -1250,7 +1250,7 @@ def is_cdbs_name(name): def is_mapping(mapping): """Return True IFF `mapping` has an extension indicating a CRDS mapping file.""" - return isinstance(mapping, str) and mapping.endswith((".pmap", ".imap", ".rmap")) + return isinstance(mapping, str) and (mapping.endswith((".pmap", ".imap", ".rmap")) or mapping.lower() in ["latest", "build"]) def is_simple_crds_mapping(mapping): """Return True IFF `mapping` implicitly or explicitly exists in the CRDS cache. @@ -1361,7 +1361,7 @@ def is_mapping_spec(mapping): def is_context(mapping): """Return True IFF `mapping` has an extension indicating a CRDS CONTEXT, i.e. .pmap.""" - return isinstance(mapping, str) and mapping.endswith((".pmap",)) + return isinstance(mapping, str) and (mapping.endswith((".pmap",)) or mapping in ["latest", "build"]) def is_context_spec(mapping): """Return True IFF `mapping` is a mapping name *or* a date based mapping specification. diff --git a/crds/core/heavy_client.py b/crds/core/heavy_client.py index bc8c505b1..37f5c95a2 100644 --- a/crds/core/heavy_client.py +++ b/crds/core/heavy_client.py @@ -422,11 +422,17 @@ def get_final_context(info, context): """ env_context = config.get_crds_env_context() if context: # context parameter trumps all, -operational is default - input_context = context + if context == 'latest': + input_context = str(info.operational_context) + else: + input_context = context log.verbose("Using reference file selection rules", srepr(input_context), "defined by caller.") info.status = "context parameter" elif env_context: - input_context = env_context + if env_context == 'latest': + input_context = str(info.operational_context) + else: + input_context = env_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by environment CRDS_CONTEXT.") info.status = "env var CRDS_CONTEXT" @@ -453,12 +459,15 @@ def translate_date_based_context(context, observatory=None): info = get_config_info(observatory) - if context == info.observatory + "-operational": + if context == info.observatory + "-operational" or context == "latest": return info["operational_context"] elif context == info.observatory + "-edit": return info["edit_context"] elif context == info.observatory + "-versions": return info["versions_context"] + elif context == "build": + #TODO + pass if not info.connected: raise CrdsError("Specified CRDS context by date '{}' and CRDS server is not reachable.".format(context)) From 1a6334f5c3711a8070d99d09f005c0765dd890ed Mon Sep 17 00:00:00 2001 From: Hunter Brown Date: Tue, 10 Sep 2024 14:05:52 -0400 Subject: [PATCH 03/22] Added function to obtain version of jwst calibration software. --- crds/client/api.py | 116 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 21 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 03cc652f8..9359f38af 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -8,6 +8,7 @@ import re import zlib import html +import importlib.metadata from urllib import request import warnings import json @@ -44,7 +45,7 @@ "list_mappings", "list_references", - "get_url", # deprecated + "get_url", # deprecated "get_flex_uri", "get_file_info", "get_file_info_map", @@ -78,7 +79,7 @@ "jpoll_abort", "get_system_versions", - ] +] # ============================================================================ @@ -86,7 +87,8 @@ URL_SUFFIX = "/json/" -S = None # Proxy server +S = None # Proxy server + def set_crds_server(url): """Configure the CRDS JSON services server to `url`, @@ -100,6 +102,7 @@ def set_crds_server(url): URL = url + URL_SUFFIX S = CheckingProxy(URL, version="1.0") + def get_crds_server(): """Return the base URL for the CRDS JSON RPC server. """ @@ -108,8 +111,10 @@ def get_crds_server(): log.warning("CRDS_SERVER_URL does not start with https:// ::", url) return url + # ============================================================================= + @utils.cached def list_mappings(observatory=None, glob_pattern="*"): """Return the list of mappings associated with `observatory` @@ -117,6 +122,7 @@ def list_mappings(observatory=None, glob_pattern="*"): """ return [str(x) for x in S.list_mappings(observatory, glob_pattern)] + @utils.cached def list_references(observatory=None, glob_pattern="*"): """Return the list of references associated with `observatory` @@ -124,6 +130,7 @@ def list_references(observatory=None, glob_pattern="*"): """ return [str(x) for x in S.list_references(observatory, glob_pattern)] + def get_mapping_url(pipeline_context, mapping): """Returns a URL for the specified pmap, imap, or rmap file. DEPRECATED """ @@ -131,6 +138,7 @@ def get_mapping_url(pipeline_context, mapping): "crds.client.get_mapping_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_mapping_url(pipeline_context, mapping) + def is_known_mapping(mapping): """Return True iff `mapping` is a known/official CRDS mapping file.""" try: @@ -138,6 +146,7 @@ def is_known_mapping(mapping): except ServiceError: return False + @utils.cached def get_mapping_names(pipeline_context): """Get the complete set of pmap, imap, and rmap basenames required @@ -146,6 +155,7 @@ def get_mapping_names(pipeline_context): """ return [str(x) for x in S.get_mapping_names(pipeline_context)] + def get_reference_url(pipeline_context, reference): """Returns a URL for the specified reference file. DEPRECATED """ @@ -153,6 +163,7 @@ def get_reference_url(pipeline_context, reference): "crds.client.get_reference_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_reference_url(pipeline_context, reference) + def get_url(pipeline_context, filename): """Return the URL for a CRDS reference or mapping file. DEPRECATED """ @@ -160,6 +171,7 @@ def get_url(pipeline_context, filename): "crds.client.get_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_url(pipeline_context, filename) + def get_flex_uri(filename, observatory=None): """If environment variables define the base URI for `filename`, append filename and return the combined URI. @@ -192,20 +204,23 @@ def get_flex_uri(filename, observatory=None): uri += filename return uri + def _unpack_info(info, section, observatory): """Return info[section][observatory] if info[section] is defined. Otherwise return "none". """ sect = info.get(section) if sect: - return sect[observatory] + return sect[observatory] else: return "none" + def get_file_info(pipeline_context, filename): """Return a dictionary of CRDS information about `filename`.""" return S.get_file_info(pipeline_context, filename) + def get_file_info_map(observatory, files=None, fields=None): """Return the info { filename : { info } } on `files` of `observatory`. `fields` can be used to limit info returned to specified keys. @@ -216,12 +231,20 @@ def get_file_info_map(observatory, files=None, fields=None): fields = tuple(sorted(fields)) return _get_file_info_map(observatory, files, fields) + @utils.cached def _get_file_info_map(observatory, files, fields): """Memory cached version of get_file_info_map() service.""" infos = S.get_file_info_map(observatory, files, fields) return infos + +def get_jwst_cal(): + """Return the version of jwst code.""" + cal_version = importlib.metadata.version('jwst') + return cal_version + + def get_total_bytes(info_map): """Return the total byte count of file info map `info_map`.""" try: @@ -230,6 +253,7 @@ def get_total_bytes(info_map): log.error("Error computing total byte count: ", str(exc)) return -1 + def get_sqlite_db(observatory): """Download the CRDS database as a SQLite database.""" assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done." @@ -241,6 +265,7 @@ def get_sqlite_db(observatory): db_out.write(data) return path + @utils.cached def get_reference_names(pipeline_context): """Get the complete set of reference file basenames required @@ -248,6 +273,7 @@ def get_reference_names(pipeline_context): """ return [str(x) for x in S.get_reference_names(pipeline_context)] + def get_best_references(pipeline_context, header, reftypes=None): """Get best references for dict-like `header` relative to `pipeline_context`. @@ -261,13 +287,14 @@ def get_best_references(pipeline_context, header, reftypes=None): Raises CrdsLookupError, typically for problems with header values """ - header = { str(key):str(value) for (key,value) in header.items() } + header = {str(key): str(value) for (key, value) in header.items()} try: bestrefs = S.get_best_references(pipeline_context, dict(header), reftypes) except Exception as exc: raise CrdsLookupError(str(exc)) from exc return bestrefs + def get_best_references_by_ids(context, dataset_ids, reftypes=None, include_headers=False): """Get best references for the specified `dataset_ids` and reference types. If reftypes is None, all types are returned. @@ -280,6 +307,7 @@ def get_best_references_by_ids(context, dataset_ids, reftypes=None, include_head raise CrdsLookupError(str(exc)) from exc return bestrefs + def get_best_references_by_header_map(context, header_map, reftypes=None): """Get best references for header_map = { dataset_id : header, ...}, } and reference types where a header is a dictionary of matching parameters. @@ -307,6 +335,7 @@ def get_aui_best_references(date, dataset_ids): raise CrdsLookupError(str(exc)) from exc return bestrefs_map + @utils.cached def get_default_context(observatory=None, state="operational"): """Return the name of the latest pipeline mapping in use for processing @@ -314,11 +343,13 @@ def get_default_context(observatory=None, state="operational"): """ return str(S.get_default_context(observatory, state)) + @utils.cached def get_context_by_date(date, observatory=None): """Return the name of the first operational context which precedes `date`.""" return str(S.get_context_by_date(date, observatory)) + @utils.cached def get_server_info(): """Return a dictionary of critical parameters about the server such as: @@ -361,12 +392,14 @@ def get_server_info(): info["download_metadata"] = proxy.crds_encode(metadata) return info + @utils.cached def get_download_metadata(): "Defer and cache decoding of download_metadata field of server info.""" info = get_server_info() return proxy.crds_decode(info["download_metadata"]) + def _get_server_info(): """Fetch the server info dict. If CRDS_CONFIG_URI is set then download that URL and load json from the contents. Otherwise, @@ -394,23 +427,28 @@ def _get_server_info(): srepr(exc)) from exc return info + get_cached_server_info = get_server_info + def get_server_version(): """Return the API version of the current CRDS server.""" info = get_server_info() return info["crds_version"]["str"] + def get_dataset_headers_by_id(context, dataset_ids, datasets_since=None): """Return { dataset_id : { header } } for `dataset_ids`.""" context = os.path.basename(context) return S.get_dataset_headers_by_id(context, dataset_ids, datasets_since) + def get_dataset_ids(context, instrument, datasets_since=None): """Return [ dataset_id, ...] for `instrument`.""" context = os.path.basename(context) return S.get_dataset_ids(context, instrument, datasets_since) + @utils.cached def get_required_parkeys(context): """Return a mapping from instruments to lists of parameter names required to @@ -421,12 +459,14 @@ def get_required_parkeys(context): context = os.path.basename(context) return S.get_required_parkeys(context) + def get_dataset_headers_by_instrument(context, instrument, datasets_since=None): """return { dataset_id:header, ...} for every `dataset_id` for `instrument`.""" log.verbose("Dumping datasets for", repr(instrument)) ids = get_dataset_ids(context, instrument, datasets_since) return dict(get_dataset_headers_unlimited(context, ids)) + def get_dataset_headers_unlimited(context, ids): """Generate (dataset_id, header) for `ids`, potentially more `ids` than can be serviced with a single JSONRPC request. @@ -435,16 +475,18 @@ def get_dataset_headers_unlimited(context, ids): """ max_ids_per_rpc = get_server_info().get("max_headers_per_rpc", 500) for i in range(0, len(ids), max_ids_per_rpc): - log.verbose("Dumping dataset headers", i , "of", len(ids), verbosity=20) - id_slice = ids[i : i + max_ids_per_rpc] + log.verbose("Dumping dataset headers", i, "of", len(ids), verbosity=20) + id_slice = ids[i: i + max_ids_per_rpc] header_slice = get_dataset_headers_by_id(context, id_slice) for item in header_slice.items(): yield item + def get_affected_datasets(observatory, old_context=None, new_context=None): """Return a structure describing the ids affected by the last context change.""" return utils.Struct(S.get_affected_datasets(observatory, old_context, new_context)) + def get_context_history(observatory): """Fetch the history of context transitions, a list of history era tuples: @@ -452,6 +494,7 @@ def get_context_history(observatory): """ return sorted(tuple(x) for x in S.get_context_history(observatory)) + def push_remote_context(observatory, kind, key, context): """Upload the specified `context` of type `kind` (e.g. "operational") to the server, informing the server of the actual configuration of the local cache @@ -465,6 +508,7 @@ def push_remote_context(observatory, kind, key, context): "Server error setting pipeline context", (observatory, kind, key, context)) from exc + def get_remote_context(observatory, pipeline_name): """Get the name of the default context last pushed from `pipeline_name` and presumed to be operational. @@ -476,8 +520,10 @@ def get_remote_context(observatory, pipeline_name): "Server error resolving context in use by pipeline", (observatory, pipeline_name)) from exc + # ============================================================================== + def jpoll_pull_messages(key, since_id=None): """Return a list of jpoll json message objects from the channel associated with `key` sent after datetime string `since` or since the last pull if @@ -490,12 +536,15 @@ def jpoll_pull_messages(key, since_id=None): messages.append(decoded) return messages + def jpoll_abort(key): """Request that the process writing to jpoll terminate on its next write.""" return S.jpoll_abort(key) + # ============================================================================== + def get_system_versions(master_version, context=None): """Return the versions Struct associated with cal s/w master_version as defined by `context` which can be defined as "null", "none", or None to use @@ -503,10 +552,13 @@ def get_system_versions(master_version, context=None): """ return utils.Struct(S.get_system_versions(master_version, str(context))) + # ============================================================================== + HARD_DEFAULT_OBS = "jwst" + def get_server_observatory(): """Return the default observatory according to the server, or None.""" try: @@ -517,6 +569,7 @@ def get_server_observatory(): server_obs = observatory_from_string(pmap) return server_obs + def get_default_observatory(): """Based on the environment, cache, and server, determine the default observatory. @@ -529,8 +582,9 @@ def get_default_observatory(): if obs != "none": return obs return observatory_from_string(get_crds_server()) or \ - get_server_observatory() or \ - "jwst" + get_server_observatory() or \ + "jwst" + def observatory_from_string(string): """If an observatory name is in `string`, return it, otherwise return None.""" @@ -539,23 +593,28 @@ def observatory_from_string(string): return observatory return None + # ============================================================================== + def file_progress(activity, name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files): """Output progress information for `activity` on file `name` at `path`.""" return "{activity} {path!s:<55} {bytes} bytes ({nth_file} / {total_files} files) ({bytes_so_far} / {total_bytes} bytes)".format( activity=activity, path=path, bytes=utils.human_format_number(bytes), - nth_file=nth_file+1, + nth_file=nth_file + 1, total_files=total_files, bytes_so_far=utils.human_format_number(bytes_so_far).strip(), total_bytes=utils.human_format_number(total_bytes).strip()) + # ============================================================================== + class FileCacher: """FileCacher gets remote files with simple names into a local cache.""" + def __init__(self, pipeline_context, ignore_cache=False, raise_exceptions=True): self.pipeline_context = pipeline_context self.observatory = self.observatory_from_context() @@ -576,7 +635,7 @@ def get_local_files(self, names): names2 = names[:] for refname in names2: if re.match(r"\w+\.r[0-9]h$", refname): - names.append(refname[:-1]+"d") + names.append(refname[:-1] + "d") downloads = [] for name in names: @@ -621,7 +680,8 @@ def download_files(self, downloads, localpaths): self.info_map = {} for filename in downloads: self.info_map[filename] = download_metadata.get(filename, "NOT FOUND unknown to server") - if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), verbosity=70): + if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), + verbosity=70): bytes_so_far = 0 total_files = len(downloads) total_bytes = get_total_bytes(self.info_map) @@ -630,7 +690,8 @@ def download_files(self, downloads, localpaths): if "NOT FOUND" in self.info_map[name]: raise CrdsDownloadError("file is not known to CRDS server.") bytes, path = self.catalog_file_size(name), localpaths[name] - log.info(file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files)) + log.info( + file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files)) self.download(name, path) bytes_so_far += os.stat(path).st_size except Exception as exc: @@ -658,7 +719,7 @@ def download(self, name, localpath): "at CRDS server", srepr(get_crds_server()), "with mode", srepr(config.get_download_mode()), ":", str(exc)) from exc - except: # mainly for control-c, catch it and throw it. + except: # mainly for control-c, catch it and throw it. self.remove_file(localpath) raise @@ -715,7 +776,8 @@ def get_data_http(self, filename): stats.increment("bytes", len(data)) status = stats.status("bytes") bytes_so_far = " ".join(status[0].split()[:-1]) - log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], verbosity=20) + log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], + verbosity=20) yield data data = infile.read(config.CRDS_DATA_CHUNK_SIZE) except Exception as exc: @@ -725,7 +787,7 @@ def get_data_http(self, filename): finally: try: infile.close() - except UnboundLocalError: # maybe the open failed. + except UnboundLocalError: # maybe the open failed. pass def get_url(self, filename): @@ -754,8 +816,10 @@ def verify_file(self, filename, localpath): else: log.verbose("Skipping sha1sum check since server doesn't know it.") + # ============================================================================== + def dump_mappings3(pipeline_context, ignore_cache=False, mappings=None, raise_exceptions=True): """Given a `pipeline_context`, determine the closure of CRDS mappings for it and cache them on the local file system. @@ -770,6 +834,7 @@ def dump_mappings3(pipeline_context, ignore_cache=False, mappings=None, raise_ex mappings = list(reversed(sorted(set(mappings)))) return FileCacher(pipeline_context, ignore_cache, raise_exceptions).get_local_files(mappings) + def dump_mappings(*args, **keys): """See dump_mappings3. @@ -777,6 +842,7 @@ def dump_mappings(*args, **keys): """ return dump_mappings3(*args, **keys)[0] + def dump_references3(pipeline_context, baserefs=None, ignore_cache=False, raise_exceptions=True): """Given a pipeline `pipeline_context` and list of `baserefs` reference file basenames, obtain the set of reference files and cache them on the @@ -796,6 +862,7 @@ def dump_references3(pipeline_context, baserefs=None, ignore_cache=False, raise_ baserefs = sorted(set(baserefs)) return FileCacher(pipeline_context, ignore_cache, raise_exceptions).get_local_files(baserefs) + def dump_references(*args, **keys): """See dump_references3. @@ -803,6 +870,7 @@ def dump_references(*args, **keys): """ return dump_references3(*args, **keys)[0] + def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exceptions=True): """Unified interface to dump any file in `files`, mapping or reference. @@ -812,8 +880,8 @@ def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exce pipeline_context = get_default_context() if files is None: files = get_mapping_names(pipeline_context) - mappings = [ os.path.basename(name) for name in files if config.is_mapping(name) ] - references = [ os.path.basename(name) for name in files if not config.is_mapping(name) ] + mappings = [os.path.basename(name) for name in files if config.is_mapping(name)] + references = [os.path.basename(name) for name in files if not config.is_mapping(name)] if mappings: m_paths, m_downloads, m_bytes = dump_mappings3( pipeline_context, mappings=mappings, ignore_cache=ignore_cache, raise_exceptions=raise_exceptions) @@ -824,7 +892,8 @@ def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exce pipeline_context, baserefs=references, ignore_cache=ignore_cache, raise_exceptions=raise_exceptions) else: r_paths, r_downloads, r_bytes = {}, 0, 0 - return dict(list(m_paths.items())+list(r_paths.items())), m_downloads + r_downloads, m_bytes + r_bytes + return dict(list(m_paths.items()) + list(r_paths.items())), m_downloads + r_downloads, m_bytes + r_bytes + # ===================================================================================================== @@ -837,6 +906,7 @@ def cache_best_references(pipeline_context, header, ignore_cache=False, reftypes local_paths = cache_references(pipeline_context, best_refs, ignore_cache) return local_paths + def cache_references(pipeline_context, bestrefs, ignore_cache=False): """Given a CRDS context `pipeline_context` and `bestrefs` dictionary, download missing reference files and cache them on the local file system. @@ -856,6 +926,7 @@ def cache_references(pipeline_context, bestrefs, ignore_cache=False): return refs + def _get_cache_filelist_and_report_errors(bestrefs): """Compute the list of files to download based on the `bestrefs` dictionary, skimming off and reporting errors, and raising an exception on the last error seen. @@ -891,6 +962,7 @@ def _get_cache_filelist_and_report_errors(bestrefs): raise last_error return wanted + def _squash_unicode_in_bestrefs(bestrefs, localrefs): """Given bestrefs dictionariesy `bestrefs` and `localrefs`, make sure there are no unicode strings anywhere in the keys or complex @@ -901,7 +973,7 @@ def _squash_unicode_in_bestrefs(bestrefs, localrefs): if isinstance(refname, tuple): refs[str(filetype)] = tuple([str(localrefs[name]) for name in refname]) elif isinstance(refname, dict): - refs[str(filetype)] = { name : str(localrefs[name]) for name in refname } + refs[str(filetype)] = {name: str(localrefs[name]) for name in refname} elif isinstance(refname, str): if "NOT FOUND" in refname: refs[str(filetype)] = str(refname) @@ -912,6 +984,7 @@ def _squash_unicode_in_bestrefs(bestrefs, localrefs): "Unhandled bestrefs return value type for", srepr(filetype)) return refs + # ===================================================================================================== # These functions are deprecated and only work when the full CRDS library is installed, and only for @@ -926,11 +999,12 @@ def cache_best_references_for_dataset(pipeline_context, dataset, header = get_minimum_header(pipeline_context, dataset, ignore_cache) return cache_best_references(pipeline_context, header, ignore_cache) + def get_minimum_header(context, dataset, ignore_cache=False): """Given a `dataset` and a `context`, extract relevant header information from the `dataset`. """ import crds dump_mappings(context, ignore_cache=ignore_cache) - ctx = crds.get_pickled_mapping(context) # reviewed + ctx = crds.get_pickled_mapping(context) # reviewed return ctx.get_minimum_header(dataset) From b11f406f4330628ced1a32b14c80e9ac54dda3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=20Ke=C3=AFn?= <3181182+alphasentaurii@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:04:16 -0400 Subject: [PATCH 04/22] api/replace-ops-with-latest (#1067) * revert back to v1 zendesk action * replace with latest * handle latest and build as context, flexibility for allowing operational * better checks for latest,build in jsonapi related calls * typo - should be = * fix typo --- crds/client/api.py | 51 ++++++++++++++++++++++++++++++--------- crds/core/cmdline.py | 7 +++--- crds/core/config.py | 21 +++++++++++----- crds/core/generic_tpn.py | 5 +++- crds/core/heavy_client.py | 36 ++++++++++++++++++--------- crds/list.py | 29 +++++++++++++++++----- crds/submit/submit.py | 3 ++- crds/sync.py | 26 ++++++++++++++------ 8 files changed, 131 insertions(+), 47 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 9359f38af..8d3a753e0 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -34,6 +34,7 @@ __all__ = [ "get_default_observatory", "get_default_context", + "get_build_context", "get_context_by_date", "get_server_info", "get_cached_server_info", # deprecated @@ -337,16 +338,37 @@ def get_aui_best_references(date, dataset_ids): @utils.cached -def get_default_context(observatory=None, state="operational"): +def get_default_context(observatory=None, state="latest"): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ return str(S.get_default_context(observatory, state)) +def get_build_context(observatory=None): + """If available, return the name of the build context pipeline mapping in use for processing + files for `observatory`. Initially only planned use is for jwst but other mission + calibration pipeline sw is included as a template. If no match found, returns latest + (formerly operational) context. + """ + if observatory == 'jwst': + calver = get_jwst_cal() + else: + cal = dict(roman='romancal', hst='caldp') + try: + calver = importlib.metadata.version(cal.get(observatory,'')) + except importlib.metadata.PackageNotFoundError: + calver = '' + if calver: + calver = config.simplify_version(calver) + return str(S.get_build_context(observatory, calver)) + else: + return get_default_context(observatory=observatory, state='latest') + + @utils.cached def get_context_by_date(date, observatory=None): - """Return the name of the first operational context which precedes `date`.""" + """Return the name of the first latest context which precedes `date`.""" return str(S.get_context_by_date(date, observatory)) @@ -354,7 +376,7 @@ def get_context_by_date(date, observatory=None): def get_server_info(): """Return a dictionary of critical parameters about the server such as: - operational_context - the context in use in the operational pipeline + latest_context - the latest context in use on the server edit_context - the context which was last edited, not necessarily archived or operational yet. @@ -496,22 +518,29 @@ def get_context_history(observatory): def push_remote_context(observatory, kind, key, context): - """Upload the specified `context` of type `kind` (e.g. "operational") to the - server, informing the server of the actual configuration of the local cache - for critical systems like pipelines, not average users. This lets the server - display actual versus commanded (Set Context) operational contexts for a pipeline. + """Upload the specified `context` of type `kind` (e.g. "latest") to the + server, informing the server of the actual configuration of the local cache. + This lets the server display actual versus commanded (Set Context) latest/operational contexts. """ try: return S.push_remote_context(observatory, kind, key, context) except Exception as exc: - raise CrdsRemoteContextError( - "Server error setting pipeline context", - (observatory, kind, key, context)) from exc + if kind == 'operational': + try: + return S.push_remote_context(observatory, 'latest', key, context) + except Exception as exc: + raise CrdsRemoteContextError( + "Server error setting latest context", + (observatory, 'latest', key, context)) from exc + else: + raise CrdsRemoteContextError( + "Server error setting operational context", + (observatory, kind, key, context)) from exc def get_remote_context(observatory, pipeline_name): """Get the name of the default context last pushed from `pipeline_name` and - presumed to be operational. + presumed to be latest. """ try: return S.get_remote_context(observatory, pipeline_name) diff --git a/crds/core/cmdline.py b/crds/core/cmdline.py index de29ceab4..ac5777a0b 100644 --- a/crds/core/cmdline.py +++ b/crds/core/cmdline.py @@ -342,8 +342,9 @@ def bad_files(self): @property def default_context(self): - """Return the default operational .pmap defined by the CRDS server or cache.""" - return self.server_info["operational_context"] + """Return the default latest .pmap defined by the CRDS server or cache.""" + default = self.server_info.get("latest_context", "operational_context") + return self.server_info[default] def get_words(self, word_list): """Process a file list, expanding @-files into corresponding lists of @@ -467,7 +468,7 @@ def run(self, *args, **keys): def resolve_context(self, context): """Resolve context spec `context` into a .pmap, .imap, or .rmap filename, interpreting - date based specifications against the CRDS server operational context history. + date based specifications against the CRDS server latest context history. """ if isinstance(context, str) and context.lower() == "none": return None diff --git a/crds/core/config.py b/crds/core/config.py index 4e6169711..c1efc544d 100644 --- a/crds/core/config.py +++ b/crds/core/config.py @@ -1211,7 +1211,7 @@ def is_cdbs_name(name): r"(?P" + r"(" + CONTEXT_DATETIME_RE_STR + r")" + r"|" + - r"(" + "edit|operational|versions" + r")" + + r"(" + "edit|operational|latest|build|versions" + r")" + r")" + r")" + r")" + @@ -1231,7 +1231,7 @@ def is_cdbs_name(name): r"(" + r"(?P" + CONTEXT_DATETIME_RE_STR + r")" + "|" + - r"(?P" + "edit|operational|versions" + r")" + + r"(?P" + "edit|operational|latest|build|versions" + r")" + r")" + r")" + r")" @@ -1250,7 +1250,7 @@ def is_cdbs_name(name): def is_mapping(mapping): """Return True IFF `mapping` has an extension indicating a CRDS mapping file.""" - return isinstance(mapping, str) and (mapping.endswith((".pmap", ".imap", ".rmap")) or mapping.lower() in ["latest", "build"]) + return isinstance(mapping, str) and mapping.endswith((".pmap", ".imap", ".rmap")) def is_simple_crds_mapping(mapping): """Return True IFF `mapping` implicitly or explicitly exists in the CRDS cache. @@ -1348,7 +1348,16 @@ def is_mapping_spec(mapping): >>> is_mapping_spec("hst-cos-deadtab-edit") True - >>> is_mapping_spec("jwst-operational") + >>> is_mapping_spec("jwst-latest") + True + + >>> is_mapping_spec("latest") + True + + >>> is_mapping_spec("jwst-build") + True + + >>> is_mapping_spec("build") True >>> is_mapping_spec("hst-foo") @@ -1357,11 +1366,11 @@ def is_mapping_spec(mapping): >>> is_mapping_spec("hst_wfc3_0001.imap") True """ - return is_mapping(mapping) or (isinstance(mapping, str) and bool(CONTEXT_RE.match(mapping))) + return is_mapping(mapping) or (isinstance(mapping, str) and bool(CONTEXT_RE.match(mapping))) or mapping in ["latest", "build"] def is_context(mapping): """Return True IFF `mapping` has an extension indicating a CRDS CONTEXT, i.e. .pmap.""" - return isinstance(mapping, str) and (mapping.endswith((".pmap",)) or mapping in ["latest", "build"]) + return isinstance(mapping, str) and mapping.endswith((".pmap",)) def is_context_spec(mapping): """Return True IFF `mapping` is a mapping name *or* a date based mapping specification. diff --git a/crds/core/generic_tpn.py b/crds/core/generic_tpn.py index 09e611c2c..af52855d4 100644 --- a/crds/core/generic_tpn.py +++ b/crds/core/generic_tpn.py @@ -292,7 +292,10 @@ def load_all_type_constraints(observatory): to customize the more generalized constraints loaded later. """ from crds.core import rmap, heavy_client - pmap_name = heavy_client.load_server_info(observatory).operational_context + try: + pmap_name = heavy_client.load_server_info(observatory).latest_context + except AttributeError: + pmap_name = heavy_client.load_server_info(observatory).operational_context pmap = rmap.get_cached_mapping(pmap_name) locator = utils.get_locator_module(observatory) for instr in pmap.selections: diff --git a/crds/core/heavy_client.py b/crds/core/heavy_client.py index 37f5c95a2..6beee8440 100644 --- a/crds/core/heavy_client.py +++ b/crds/core/heavy_client.py @@ -404,7 +404,7 @@ def get_context_name(observatory, context=None): """Return the .pmap name of the default context based on: 1. literal definitiion in `context` (jwst_0001.pmap) - 2. symbolic definition in `context` (jwst-operational or jwst-edit) + 2. symbolic definition in `context` (latest or jwst-edit) 3. date-based definition in `context` (jwst-2017-01-15T00:05:00) 4. CRDS_CONTEXT env var override @@ -415,29 +415,42 @@ def get_context_name(observatory, context=None): def get_final_context(info, context): """Based on env CRDS_CONTEXT, the `context` parameter, and the server's reported, - cached, or defaulted `operational_context`, choose the pipeline mapping which + cached, or defaulted `latest_context`, choose the pipeline mapping which defines the reference selection rules. Returns a .pmap name """ env_context = config.get_crds_env_context() - if context: # context parameter trumps all, -operational is default + try: + latest_context = str(info.latest_context) + except AttributeError: + latest_context = str(info.operational_context) + if context: # context parameter trumps all, -latest is default if context == 'latest': - input_context = str(info.operational_context) + input_context = latest_context + elif context == 'build': + input_context = api.get_build_context(info.observatory) else: input_context = context log.verbose("Using reference file selection rules", srepr(input_context), "defined by caller.") info.status = "context parameter" elif env_context: - if env_context == 'latest': - input_context = str(info.operational_context) + if env_context in ['latest', f"{info.observatory}-operational", f"{info.observatory}-latest"]: + input_context = latest_context + elif env_context in ['build', f"{info.observatory}-build"]: + input_context = api.get_build_context(info.observatory) else: input_context = env_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by environment CRDS_CONTEXT.") info.status = "env var CRDS_CONTEXT" else: - input_context = str(info.operational_context) + # Default for JWST if no env context and no explicit context is BUILD CONTEXT for installed jwst version + if info.observatory == 'jwst': + input_context = api.get_build_context('jwst') + # For other missions, default is latest (formerly operational) context + else: + input_context = latest_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by", info.status + ".") final_context = translate_date_based_context(input_context, info.observatory) return final_context @@ -459,15 +472,14 @@ def translate_date_based_context(context, observatory=None): info = get_config_info(observatory) - if context == info.observatory + "-operational" or context == "latest": - return info["operational_context"] + latest_context_names = [info.observatory + "-operational", info.observatory + "-latest", "latest"] + + if context in latest_context_names: + return info.get('latest_context', info['operational_context']) elif context == info.observatory + "-edit": return info["edit_context"] elif context == info.observatory + "-versions": return info["versions_context"] - elif context == "build": - #TODO - pass if not info.connected: raise CrdsError("Specified CRDS context by date '{}' and CRDS server is not reachable.".format(context)) diff --git a/crds/list.py b/crds/list.py index 5ac294981..0fb3c59b4 100644 --- a/crds/list.py +++ b/crds/list.py @@ -348,7 +348,10 @@ def add_args(self): self.add_argument("--tpns", nargs="*", dest="tpns", metavar="FILES", default=None, help="print the certify constraints (.tpn's) associated with the specified or implied files.") - + self.add_argument("--latest-context", action="store_true", dest="latest_context", + help="print the name of the latest context on the central CRDS server.") + self.add_argument("--build-context", action="store_true", dest="build_context", + help="print the name of the build context on the central CRDS server.") self.add_argument("--operational-context", action="store_true", dest="operational_context", help="print the name of the operational context on the central CRDS server.") self.add_argument("--remote-context", type=str, metavar="PIPELINE", @@ -375,12 +378,15 @@ def main(self): if self.args.file_properties is not None: # including [] return self.list_file_properties() - if self.args.operational_context: + if self.args.operational_context or self.args.latest_context: print(self.default_context) return if self.args.remote_context: print(self.remote_context) return + if self.args.build_context: + print(self.build_context) + return if self.args.resolve_contexts: self.list_resolved_contexts() @@ -410,7 +416,7 @@ def main(self): self.list_required_parkeys() return log.errors() - + def list_resolved_contexts(self): """Print out the literal interpretation of the contexts implied by the script's context specifiers. @@ -428,6 +434,12 @@ def remote_context(self): with log.error_on_exception("Failed resolving remote context"): return api.get_remote_context(self.observatory, self.args.remote_context) + @property + def build_context(self): + self.require_server_connection() + with log.error_on_exception("Failed resolving build context"): + return api.get_build_context(self.observatory) + @property def implied_files(self): """Return the filenames implied by a combination of the specified contexts @@ -656,9 +668,14 @@ def list_config(self): "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) - _print_dict("CRDS Server Info", server, - ["observatory", "status", "connected", "operational_context", "last_synced", - "reference_url", "mapping_url", "effective_mode"]) + try: + _print_dict("CRDS Server Info", server, + ["observatory", "status", "connected", "latest_context", "last_synced", + "reference_url", "mapping_url", "effective_mode"]) + except Exception: + _print_dict("CRDS Server Info", server, + ["observatory", "status", "connected", "operational_context", "last_synced", + "reference_url", "mapping_url", "effective_mode"]) download_uris = dict( config_uri = os.path.dirname(api.get_flex_uri("server_config")), pickle_uri = os.path.dirname(api.get_flex_uri("xyz.pmap.pkl")), diff --git a/crds/submit/submit.py b/crds/submit/submit.py index 26bdb9d9f..f317b22bc 100644 --- a/crds/submit/submit.py +++ b/crds/submit/submit.py @@ -106,7 +106,8 @@ def finish_parameters(self): locked_instrument=self.instrument, username=self.username, password=password, base_url=self.base_url) if self.args.derive_from_context is None: self.args.derive_from_context = self.observatory + "-edit" - if self.args.derive_from_context.endswith(("-edit", "-operational")): + self.args.derive_from_context.replace("-operational", "-latest") + if self.args.derive_from_context.endswith(("-edit", "-latest")): # this actually floats for concurrent deliveries self.pmap_mode = "pmap_" + self.args.derive_from_context.split("-")[-1] else: diff --git a/crds/sync.py b/crds/sync.py index 2be0d8706..c1d121ce7 100644 --- a/crds/sync.py +++ b/crds/sync.py @@ -401,7 +401,7 @@ def get_synced_references(self): return active_references def update_context(self): - """Update the CRDS operational context in the cache. Handle pipeline-specific + """Update the CRDS latest context in the cache. Handle pipeline-specific targeted features of (a) verifying a context switch as actually recorded in the local CRDS cache and (b) echoing/pushing the pipeline context back up to the CRDS server for tracking using an id/authorization key. @@ -410,7 +410,10 @@ def update_context(self): """ if not log.errors() or self.args.force_config_update: if self.args.verify_context_change: - old_context = heavy_client.load_server_info(self.observatory).operational_context + try: + old_context = heavy_client.load_server_info(self.observatory).latest_context + except AttributeError: + old_context = heavy_client.load_server_info(self.observatory).operational_context heavy_client.update_config_info(self.observatory) if self.args.verify_context_change: self.verify_context_change(old_context) @@ -444,7 +447,10 @@ def server_info(self): def verify_context_change(self, old_context): """Verify that the starting and post-sync contexts are different, or issue an error.""" - new_context = heavy_client.load_server_info(self.observatory).operational_context + try: + new_context = heavy_client.load_server_info(self.observatory).latest_context + except AttributeError: + new_context = heavy_client.load_server_info(self.observatory).operational_context if old_context == new_context: log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context)) else: @@ -452,12 +458,18 @@ def verify_context_change(self, old_context): def push_context(self): """Push the final context recorded in the local cache to the CRDS server so it can be displayed - as the operational state of a pipeline. + as the latest state of a pipeline. """ info = heavy_client.load_server_info(self.observatory) - with log.error_on_exception("Failed pushing cached operational context name to CRDS server"): - api.push_remote_context(self.observatory, "operational", self.args.push_context, info.operational_context) - log.info("Pushed cached operational context name", repr(info.operational_context), "to CRDS server") + try: + ctx = 'latest' + latest_context = info.latest_context + except AttributeError: + ctx = 'operational' + latest_context = info.operational_context + with log.error_on_exception(f"Failed pushing cached {ctx} context name to CRDS server"): + api.push_remote_context(self.observatory, ctx, self.args.push_context, latest_context) + log.info(f"Pushed cached {ctx} context name", repr(latest_context), "to CRDS server") # ------------------------------------------------------------------------------------------ From 9126f8c7a63df487259bef86e0a1dbf90c52e137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=20Ke=C3=AFn?= <3181182+alphasentaurii@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:50:05 -0400 Subject: [PATCH 05/22] buildcontext/fallback (#1070) * revert back to v1 zendesk action * if no cal software installed, default to latest --- crds/client/api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 8d3a753e0..440283797 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -351,18 +351,20 @@ def get_build_context(observatory=None): calibration pipeline sw is included as a template. If no match found, returns latest (formerly operational) context. """ - if observatory == 'jwst': - calver = get_jwst_cal() - else: - cal = dict(roman='romancal', hst='caldp') - try: + try: + if observatory == 'jwst': + calver = get_jwst_cal() + else: + cal = dict(roman='romancal', hst='caldp') calver = importlib.metadata.version(cal.get(observatory,'')) - except importlib.metadata.PackageNotFoundError: - calver = '' + except importlib.metadata.PackageNotFoundError: + calver = '' + log.warning("Cal SW not found, defaulting to latest.") if calver: calver = config.simplify_version(calver) return str(S.get_build_context(observatory, calver)) else: + return get_default_context(observatory=observatory, state='latest') From 93a3f6848f0647a0cd112ffdca3efcb801889740 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Mon, 16 Sep 2024 16:49:16 -0400 Subject: [PATCH 06/22] generalize calver lookup, handle various cases --- crds/client/api.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 440283797..1343f831d 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -240,9 +240,17 @@ def _get_file_info_map(observatory, files, fields): return infos -def get_jwst_cal(): - """Return the version of jwst code.""" - cal_version = importlib.metadata.version('jwst') +def get_cal_version(observatory): + """Return the version of observatory calibration software.""" + cal_version = '' + if observatory: + cal = dict(jwst='jwst', roman='romancal', hst='caldp')[observatory] + try: + cal_version = importlib.metadata.version(cal) + calver = config.simplify_version(calver) + log.info(f"Calibration SW Found: {cal} {calver}") + except importlib.metadata.PackageNotFoundError: + log.warning("Calibration SW not found, defaulting to latest.") return cal_version @@ -342,6 +350,8 @@ def get_default_context(observatory=None, state="latest"): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ + if state == "build": + return get_build_context(observatory=observatory) return str(S.get_default_context(observatory, state)) @@ -351,20 +361,15 @@ def get_build_context(observatory=None): calibration pipeline sw is included as a template. If no match found, returns latest (formerly operational) context. """ - try: - if observatory == 'jwst': - calver = get_jwst_cal() - else: - cal = dict(roman='romancal', hst='caldp') - calver = importlib.metadata.version(cal.get(observatory,'')) - except importlib.metadata.PackageNotFoundError: - calver = '' - log.warning("Cal SW not found, defaulting to latest.") + if observatory is None: + try: + observatory = str(S).split("=")[1].split("-")[0].split("/")[-1] + except ServiceError: + observatory = os.environ.get('CRDS_SERVER_URL', '').split("-")[0].split("/")[-1] + calver = get_cal_version(observatory) if calver: - calver = config.simplify_version(calver) return str(S.get_build_context(observatory, calver)) else: - return get_default_context(observatory=observatory, state='latest') From d6a91d5f4c14331a1a714244fad5928ddf3150f4 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Mon, 16 Sep 2024 16:59:58 -0400 Subject: [PATCH 07/22] fix typo cal_version not calver --- crds/client/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 1343f831d..f7c4241d3 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -247,8 +247,8 @@ def get_cal_version(observatory): cal = dict(jwst='jwst', roman='romancal', hst='caldp')[observatory] try: cal_version = importlib.metadata.version(cal) - calver = config.simplify_version(calver) - log.info(f"Calibration SW Found: {cal} {calver}") + cal_version = config.simplify_version(cal_version) + log.info(f"Calibration SW Found: {cal} {cal_version}") except importlib.metadata.PackageNotFoundError: log.warning("Calibration SW not found, defaulting to latest.") return cal_version From 75de4c71d76195f1ed85983f7de951f851e7d067 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Thu, 19 Sep 2024 11:24:26 -0400 Subject: [PATCH 08/22] ensure get_default_context with no args defaults to build for jwst, else latest --- crds/client/api.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index f7c4241d3..98dffd56e 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -346,11 +346,12 @@ def get_aui_best_references(date, dataset_ids): @utils.cached -def get_default_context(observatory=None, state="latest"): +def get_default_context(observatory=None, state=None): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ - if state == "build": + observatory = get_default_observatory() if observatory is None else observatory + if state == "build" or (observatory == "jwst" and state not in ["edit", "latest"]): return get_build_context(observatory=observatory) return str(S.get_default_context(observatory, state)) @@ -358,19 +359,18 @@ def get_default_context(observatory=None, state="latest"): def get_build_context(observatory=None): """If available, return the name of the build context pipeline mapping in use for processing files for `observatory`. Initially only planned use is for jwst but other mission - calibration pipeline sw is included as a template. If no match found, returns latest - (formerly operational) context. + calibration pipeline sw is included as a template. If exact match is not found, an attempt to + find next closest (previous) patch version is made. Ultimate fallback is to the latest + (formerly 'operational') context. """ - if observatory is None: - try: - observatory = str(S).split("=")[1].split("-")[0].split("/")[-1] - except ServiceError: - observatory = os.environ.get('CRDS_SERVER_URL', '').split("-")[0].split("/")[-1] + observatory = get_default_observatory() if observatory is None else observatory calver = get_cal_version(observatory) if calver: - return str(S.get_build_context(observatory, calver)) - else: - return get_default_context(observatory=observatory, state='latest') + try: + return str(S.get_build_context(observatory, calver)) + except ServiceError: + log.warning("Server build context could not be identified. Using 'latest' instead.") + return get_default_context(observatory, "latest") @utils.cached @@ -615,7 +615,7 @@ def get_default_observatory(): 4. jwst """ obs = config.OBSERVATORY.get() - if obs != "none": + if obs not in ["none", ""]: return obs return observatory_from_string(get_crds_server()) or \ get_server_observatory() or \ From d96fc5afb7fcb5b2f405f1f28ed4b698cc756d3b Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Thu, 19 Sep 2024 11:28:00 -0400 Subject: [PATCH 09/22] update changelog --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e69ed8dc7..84ebe6159 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,8 @@ General - Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] +- `client.api.get_default_context` by default returns build context for jwst, else latest. This can still be overridden by explicitly passing a value into optional arg `state`. [#1069] + 11.18.3 (2024-09-03) ==================== From 858e86791fa54ab2a176fd2981ab82aa67424663 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Thu, 19 Sep 2024 11:56:40 -0400 Subject: [PATCH 10/22] handle env var CRDS_OBSERVATORY empty string --- crds/client/api.py | 2 +- crds/core/cmdline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 98dffd56e..722ca4c17 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -615,7 +615,7 @@ def get_default_observatory(): 4. jwst """ obs = config.OBSERVATORY.get() - if obs not in ["none", ""]: + if obs not in ["none", "", None]: return obs return observatory_from_string(get_crds_server()) or \ get_server_observatory() or \ diff --git a/crds/core/cmdline.py b/crds/core/cmdline.py index ac5777a0b..19de075b1 100644 --- a/crds/core/cmdline.py +++ b/crds/core/cmdline.py @@ -222,7 +222,7 @@ def observatory(self): return self.set_server("roman") obs = config.OBSERVATORY.get() - if obs != "none": + if obs not in ["none", "", None]: return self.set_server(obs.lower()) url = os.environ.get("CRDS_SERVER_URL", None) From c0adfa12e8853912e2273f2b802f9e571b48b316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=20Ke=C3=AFn?= <3181182+alphasentaurii@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:54:32 -0400 Subject: [PATCH 11/22] core/allow-context-latest (#1062) * translate latest to operational context * incl state in server call to get_default_context * update changelog --- CHANGES.rst | 10 ++++++++++ crds/client/api.py | 4 ++-- crds/core/config.py | 4 ++-- crds/core/heavy_client.py | 15 ++++++++++++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 50c4b9dae..63f48bc7c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,13 @@ +12.0.0 (unreleased) +==================== + +General +------- + +- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. + +- Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] + 11.18.4 (2024-09-10) ==================== diff --git a/crds/client/api.py b/crds/client/api.py index 8d5e7c059..03cc652f8 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -308,11 +308,11 @@ def get_aui_best_references(date, dataset_ids): return bestrefs_map @utils.cached -def get_default_context(observatory=None): +def get_default_context(observatory=None, state="operational"): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ - return str(S.get_default_context(observatory)) + return str(S.get_default_context(observatory, state)) @utils.cached def get_context_by_date(date, observatory=None): diff --git a/crds/core/config.py b/crds/core/config.py index 24e95b1e0..ddf518510 100644 --- a/crds/core/config.py +++ b/crds/core/config.py @@ -1250,7 +1250,7 @@ def is_cdbs_name(name): def is_mapping(mapping): """Return True IFF `mapping` has an extension indicating a CRDS mapping file.""" - return isinstance(mapping, str) and mapping.endswith((".pmap", ".imap", ".rmap")) + return isinstance(mapping, str) and (mapping.endswith((".pmap", ".imap", ".rmap")) or mapping.lower() in ["latest", "build"]) def is_simple_crds_mapping(mapping): """Return True IFF `mapping` implicitly or explicitly exists in the CRDS cache. @@ -1361,7 +1361,7 @@ def is_mapping_spec(mapping): def is_context(mapping): """Return True IFF `mapping` has an extension indicating a CRDS CONTEXT, i.e. .pmap.""" - return isinstance(mapping, str) and mapping.endswith((".pmap",)) + return isinstance(mapping, str) and (mapping.endswith((".pmap",)) or mapping in ["latest", "build"]) def is_context_spec(mapping): """Return True IFF `mapping` is a mapping name *or* a date based mapping specification. diff --git a/crds/core/heavy_client.py b/crds/core/heavy_client.py index bc8c505b1..37f5c95a2 100644 --- a/crds/core/heavy_client.py +++ b/crds/core/heavy_client.py @@ -422,11 +422,17 @@ def get_final_context(info, context): """ env_context = config.get_crds_env_context() if context: # context parameter trumps all, -operational is default - input_context = context + if context == 'latest': + input_context = str(info.operational_context) + else: + input_context = context log.verbose("Using reference file selection rules", srepr(input_context), "defined by caller.") info.status = "context parameter" elif env_context: - input_context = env_context + if env_context == 'latest': + input_context = str(info.operational_context) + else: + input_context = env_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by environment CRDS_CONTEXT.") info.status = "env var CRDS_CONTEXT" @@ -453,12 +459,15 @@ def translate_date_based_context(context, observatory=None): info = get_config_info(observatory) - if context == info.observatory + "-operational": + if context == info.observatory + "-operational" or context == "latest": return info["operational_context"] elif context == info.observatory + "-edit": return info["edit_context"] elif context == info.observatory + "-versions": return info["versions_context"] + elif context == "build": + #TODO + pass if not info.connected: raise CrdsError("Specified CRDS context by date '{}' and CRDS server is not reachable.".format(context)) From 85ce5669d5d4e03a75625a25704b7abe96c089c5 Mon Sep 17 00:00:00 2001 From: Hunter Brown Date: Tue, 10 Sep 2024 14:05:52 -0400 Subject: [PATCH 12/22] Added function to obtain version of jwst calibration software. --- crds/client/api.py | 116 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 21 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 03cc652f8..9359f38af 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -8,6 +8,7 @@ import re import zlib import html +import importlib.metadata from urllib import request import warnings import json @@ -44,7 +45,7 @@ "list_mappings", "list_references", - "get_url", # deprecated + "get_url", # deprecated "get_flex_uri", "get_file_info", "get_file_info_map", @@ -78,7 +79,7 @@ "jpoll_abort", "get_system_versions", - ] +] # ============================================================================ @@ -86,7 +87,8 @@ URL_SUFFIX = "/json/" -S = None # Proxy server +S = None # Proxy server + def set_crds_server(url): """Configure the CRDS JSON services server to `url`, @@ -100,6 +102,7 @@ def set_crds_server(url): URL = url + URL_SUFFIX S = CheckingProxy(URL, version="1.0") + def get_crds_server(): """Return the base URL for the CRDS JSON RPC server. """ @@ -108,8 +111,10 @@ def get_crds_server(): log.warning("CRDS_SERVER_URL does not start with https:// ::", url) return url + # ============================================================================= + @utils.cached def list_mappings(observatory=None, glob_pattern="*"): """Return the list of mappings associated with `observatory` @@ -117,6 +122,7 @@ def list_mappings(observatory=None, glob_pattern="*"): """ return [str(x) for x in S.list_mappings(observatory, glob_pattern)] + @utils.cached def list_references(observatory=None, glob_pattern="*"): """Return the list of references associated with `observatory` @@ -124,6 +130,7 @@ def list_references(observatory=None, glob_pattern="*"): """ return [str(x) for x in S.list_references(observatory, glob_pattern)] + def get_mapping_url(pipeline_context, mapping): """Returns a URL for the specified pmap, imap, or rmap file. DEPRECATED """ @@ -131,6 +138,7 @@ def get_mapping_url(pipeline_context, mapping): "crds.client.get_mapping_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_mapping_url(pipeline_context, mapping) + def is_known_mapping(mapping): """Return True iff `mapping` is a known/official CRDS mapping file.""" try: @@ -138,6 +146,7 @@ def is_known_mapping(mapping): except ServiceError: return False + @utils.cached def get_mapping_names(pipeline_context): """Get the complete set of pmap, imap, and rmap basenames required @@ -146,6 +155,7 @@ def get_mapping_names(pipeline_context): """ return [str(x) for x in S.get_mapping_names(pipeline_context)] + def get_reference_url(pipeline_context, reference): """Returns a URL for the specified reference file. DEPRECATED """ @@ -153,6 +163,7 @@ def get_reference_url(pipeline_context, reference): "crds.client.get_reference_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_reference_url(pipeline_context, reference) + def get_url(pipeline_context, filename): """Return the URL for a CRDS reference or mapping file. DEPRECATED """ @@ -160,6 +171,7 @@ def get_url(pipeline_context, filename): "crds.client.get_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_url(pipeline_context, filename) + def get_flex_uri(filename, observatory=None): """If environment variables define the base URI for `filename`, append filename and return the combined URI. @@ -192,20 +204,23 @@ def get_flex_uri(filename, observatory=None): uri += filename return uri + def _unpack_info(info, section, observatory): """Return info[section][observatory] if info[section] is defined. Otherwise return "none". """ sect = info.get(section) if sect: - return sect[observatory] + return sect[observatory] else: return "none" + def get_file_info(pipeline_context, filename): """Return a dictionary of CRDS information about `filename`.""" return S.get_file_info(pipeline_context, filename) + def get_file_info_map(observatory, files=None, fields=None): """Return the info { filename : { info } } on `files` of `observatory`. `fields` can be used to limit info returned to specified keys. @@ -216,12 +231,20 @@ def get_file_info_map(observatory, files=None, fields=None): fields = tuple(sorted(fields)) return _get_file_info_map(observatory, files, fields) + @utils.cached def _get_file_info_map(observatory, files, fields): """Memory cached version of get_file_info_map() service.""" infos = S.get_file_info_map(observatory, files, fields) return infos + +def get_jwst_cal(): + """Return the version of jwst code.""" + cal_version = importlib.metadata.version('jwst') + return cal_version + + def get_total_bytes(info_map): """Return the total byte count of file info map `info_map`.""" try: @@ -230,6 +253,7 @@ def get_total_bytes(info_map): log.error("Error computing total byte count: ", str(exc)) return -1 + def get_sqlite_db(observatory): """Download the CRDS database as a SQLite database.""" assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done." @@ -241,6 +265,7 @@ def get_sqlite_db(observatory): db_out.write(data) return path + @utils.cached def get_reference_names(pipeline_context): """Get the complete set of reference file basenames required @@ -248,6 +273,7 @@ def get_reference_names(pipeline_context): """ return [str(x) for x in S.get_reference_names(pipeline_context)] + def get_best_references(pipeline_context, header, reftypes=None): """Get best references for dict-like `header` relative to `pipeline_context`. @@ -261,13 +287,14 @@ def get_best_references(pipeline_context, header, reftypes=None): Raises CrdsLookupError, typically for problems with header values """ - header = { str(key):str(value) for (key,value) in header.items() } + header = {str(key): str(value) for (key, value) in header.items()} try: bestrefs = S.get_best_references(pipeline_context, dict(header), reftypes) except Exception as exc: raise CrdsLookupError(str(exc)) from exc return bestrefs + def get_best_references_by_ids(context, dataset_ids, reftypes=None, include_headers=False): """Get best references for the specified `dataset_ids` and reference types. If reftypes is None, all types are returned. @@ -280,6 +307,7 @@ def get_best_references_by_ids(context, dataset_ids, reftypes=None, include_head raise CrdsLookupError(str(exc)) from exc return bestrefs + def get_best_references_by_header_map(context, header_map, reftypes=None): """Get best references for header_map = { dataset_id : header, ...}, } and reference types where a header is a dictionary of matching parameters. @@ -307,6 +335,7 @@ def get_aui_best_references(date, dataset_ids): raise CrdsLookupError(str(exc)) from exc return bestrefs_map + @utils.cached def get_default_context(observatory=None, state="operational"): """Return the name of the latest pipeline mapping in use for processing @@ -314,11 +343,13 @@ def get_default_context(observatory=None, state="operational"): """ return str(S.get_default_context(observatory, state)) + @utils.cached def get_context_by_date(date, observatory=None): """Return the name of the first operational context which precedes `date`.""" return str(S.get_context_by_date(date, observatory)) + @utils.cached def get_server_info(): """Return a dictionary of critical parameters about the server such as: @@ -361,12 +392,14 @@ def get_server_info(): info["download_metadata"] = proxy.crds_encode(metadata) return info + @utils.cached def get_download_metadata(): "Defer and cache decoding of download_metadata field of server info.""" info = get_server_info() return proxy.crds_decode(info["download_metadata"]) + def _get_server_info(): """Fetch the server info dict. If CRDS_CONFIG_URI is set then download that URL and load json from the contents. Otherwise, @@ -394,23 +427,28 @@ def _get_server_info(): srepr(exc)) from exc return info + get_cached_server_info = get_server_info + def get_server_version(): """Return the API version of the current CRDS server.""" info = get_server_info() return info["crds_version"]["str"] + def get_dataset_headers_by_id(context, dataset_ids, datasets_since=None): """Return { dataset_id : { header } } for `dataset_ids`.""" context = os.path.basename(context) return S.get_dataset_headers_by_id(context, dataset_ids, datasets_since) + def get_dataset_ids(context, instrument, datasets_since=None): """Return [ dataset_id, ...] for `instrument`.""" context = os.path.basename(context) return S.get_dataset_ids(context, instrument, datasets_since) + @utils.cached def get_required_parkeys(context): """Return a mapping from instruments to lists of parameter names required to @@ -421,12 +459,14 @@ def get_required_parkeys(context): context = os.path.basename(context) return S.get_required_parkeys(context) + def get_dataset_headers_by_instrument(context, instrument, datasets_since=None): """return { dataset_id:header, ...} for every `dataset_id` for `instrument`.""" log.verbose("Dumping datasets for", repr(instrument)) ids = get_dataset_ids(context, instrument, datasets_since) return dict(get_dataset_headers_unlimited(context, ids)) + def get_dataset_headers_unlimited(context, ids): """Generate (dataset_id, header) for `ids`, potentially more `ids` than can be serviced with a single JSONRPC request. @@ -435,16 +475,18 @@ def get_dataset_headers_unlimited(context, ids): """ max_ids_per_rpc = get_server_info().get("max_headers_per_rpc", 500) for i in range(0, len(ids), max_ids_per_rpc): - log.verbose("Dumping dataset headers", i , "of", len(ids), verbosity=20) - id_slice = ids[i : i + max_ids_per_rpc] + log.verbose("Dumping dataset headers", i, "of", len(ids), verbosity=20) + id_slice = ids[i: i + max_ids_per_rpc] header_slice = get_dataset_headers_by_id(context, id_slice) for item in header_slice.items(): yield item + def get_affected_datasets(observatory, old_context=None, new_context=None): """Return a structure describing the ids affected by the last context change.""" return utils.Struct(S.get_affected_datasets(observatory, old_context, new_context)) + def get_context_history(observatory): """Fetch the history of context transitions, a list of history era tuples: @@ -452,6 +494,7 @@ def get_context_history(observatory): """ return sorted(tuple(x) for x in S.get_context_history(observatory)) + def push_remote_context(observatory, kind, key, context): """Upload the specified `context` of type `kind` (e.g. "operational") to the server, informing the server of the actual configuration of the local cache @@ -465,6 +508,7 @@ def push_remote_context(observatory, kind, key, context): "Server error setting pipeline context", (observatory, kind, key, context)) from exc + def get_remote_context(observatory, pipeline_name): """Get the name of the default context last pushed from `pipeline_name` and presumed to be operational. @@ -476,8 +520,10 @@ def get_remote_context(observatory, pipeline_name): "Server error resolving context in use by pipeline", (observatory, pipeline_name)) from exc + # ============================================================================== + def jpoll_pull_messages(key, since_id=None): """Return a list of jpoll json message objects from the channel associated with `key` sent after datetime string `since` or since the last pull if @@ -490,12 +536,15 @@ def jpoll_pull_messages(key, since_id=None): messages.append(decoded) return messages + def jpoll_abort(key): """Request that the process writing to jpoll terminate on its next write.""" return S.jpoll_abort(key) + # ============================================================================== + def get_system_versions(master_version, context=None): """Return the versions Struct associated with cal s/w master_version as defined by `context` which can be defined as "null", "none", or None to use @@ -503,10 +552,13 @@ def get_system_versions(master_version, context=None): """ return utils.Struct(S.get_system_versions(master_version, str(context))) + # ============================================================================== + HARD_DEFAULT_OBS = "jwst" + def get_server_observatory(): """Return the default observatory according to the server, or None.""" try: @@ -517,6 +569,7 @@ def get_server_observatory(): server_obs = observatory_from_string(pmap) return server_obs + def get_default_observatory(): """Based on the environment, cache, and server, determine the default observatory. @@ -529,8 +582,9 @@ def get_default_observatory(): if obs != "none": return obs return observatory_from_string(get_crds_server()) or \ - get_server_observatory() or \ - "jwst" + get_server_observatory() or \ + "jwst" + def observatory_from_string(string): """If an observatory name is in `string`, return it, otherwise return None.""" @@ -539,23 +593,28 @@ def observatory_from_string(string): return observatory return None + # ============================================================================== + def file_progress(activity, name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files): """Output progress information for `activity` on file `name` at `path`.""" return "{activity} {path!s:<55} {bytes} bytes ({nth_file} / {total_files} files) ({bytes_so_far} / {total_bytes} bytes)".format( activity=activity, path=path, bytes=utils.human_format_number(bytes), - nth_file=nth_file+1, + nth_file=nth_file + 1, total_files=total_files, bytes_so_far=utils.human_format_number(bytes_so_far).strip(), total_bytes=utils.human_format_number(total_bytes).strip()) + # ============================================================================== + class FileCacher: """FileCacher gets remote files with simple names into a local cache.""" + def __init__(self, pipeline_context, ignore_cache=False, raise_exceptions=True): self.pipeline_context = pipeline_context self.observatory = self.observatory_from_context() @@ -576,7 +635,7 @@ def get_local_files(self, names): names2 = names[:] for refname in names2: if re.match(r"\w+\.r[0-9]h$", refname): - names.append(refname[:-1]+"d") + names.append(refname[:-1] + "d") downloads = [] for name in names: @@ -621,7 +680,8 @@ def download_files(self, downloads, localpaths): self.info_map = {} for filename in downloads: self.info_map[filename] = download_metadata.get(filename, "NOT FOUND unknown to server") - if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), verbosity=70): + if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), + verbosity=70): bytes_so_far = 0 total_files = len(downloads) total_bytes = get_total_bytes(self.info_map) @@ -630,7 +690,8 @@ def download_files(self, downloads, localpaths): if "NOT FOUND" in self.info_map[name]: raise CrdsDownloadError("file is not known to CRDS server.") bytes, path = self.catalog_file_size(name), localpaths[name] - log.info(file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files)) + log.info( + file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files)) self.download(name, path) bytes_so_far += os.stat(path).st_size except Exception as exc: @@ -658,7 +719,7 @@ def download(self, name, localpath): "at CRDS server", srepr(get_crds_server()), "with mode", srepr(config.get_download_mode()), ":", str(exc)) from exc - except: # mainly for control-c, catch it and throw it. + except: # mainly for control-c, catch it and throw it. self.remove_file(localpath) raise @@ -715,7 +776,8 @@ def get_data_http(self, filename): stats.increment("bytes", len(data)) status = stats.status("bytes") bytes_so_far = " ".join(status[0].split()[:-1]) - log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], verbosity=20) + log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], + verbosity=20) yield data data = infile.read(config.CRDS_DATA_CHUNK_SIZE) except Exception as exc: @@ -725,7 +787,7 @@ def get_data_http(self, filename): finally: try: infile.close() - except UnboundLocalError: # maybe the open failed. + except UnboundLocalError: # maybe the open failed. pass def get_url(self, filename): @@ -754,8 +816,10 @@ def verify_file(self, filename, localpath): else: log.verbose("Skipping sha1sum check since server doesn't know it.") + # ============================================================================== + def dump_mappings3(pipeline_context, ignore_cache=False, mappings=None, raise_exceptions=True): """Given a `pipeline_context`, determine the closure of CRDS mappings for it and cache them on the local file system. @@ -770,6 +834,7 @@ def dump_mappings3(pipeline_context, ignore_cache=False, mappings=None, raise_ex mappings = list(reversed(sorted(set(mappings)))) return FileCacher(pipeline_context, ignore_cache, raise_exceptions).get_local_files(mappings) + def dump_mappings(*args, **keys): """See dump_mappings3. @@ -777,6 +842,7 @@ def dump_mappings(*args, **keys): """ return dump_mappings3(*args, **keys)[0] + def dump_references3(pipeline_context, baserefs=None, ignore_cache=False, raise_exceptions=True): """Given a pipeline `pipeline_context` and list of `baserefs` reference file basenames, obtain the set of reference files and cache them on the @@ -796,6 +862,7 @@ def dump_references3(pipeline_context, baserefs=None, ignore_cache=False, raise_ baserefs = sorted(set(baserefs)) return FileCacher(pipeline_context, ignore_cache, raise_exceptions).get_local_files(baserefs) + def dump_references(*args, **keys): """See dump_references3. @@ -803,6 +870,7 @@ def dump_references(*args, **keys): """ return dump_references3(*args, **keys)[0] + def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exceptions=True): """Unified interface to dump any file in `files`, mapping or reference. @@ -812,8 +880,8 @@ def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exce pipeline_context = get_default_context() if files is None: files = get_mapping_names(pipeline_context) - mappings = [ os.path.basename(name) for name in files if config.is_mapping(name) ] - references = [ os.path.basename(name) for name in files if not config.is_mapping(name) ] + mappings = [os.path.basename(name) for name in files if config.is_mapping(name)] + references = [os.path.basename(name) for name in files if not config.is_mapping(name)] if mappings: m_paths, m_downloads, m_bytes = dump_mappings3( pipeline_context, mappings=mappings, ignore_cache=ignore_cache, raise_exceptions=raise_exceptions) @@ -824,7 +892,8 @@ def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exce pipeline_context, baserefs=references, ignore_cache=ignore_cache, raise_exceptions=raise_exceptions) else: r_paths, r_downloads, r_bytes = {}, 0, 0 - return dict(list(m_paths.items())+list(r_paths.items())), m_downloads + r_downloads, m_bytes + r_bytes + return dict(list(m_paths.items()) + list(r_paths.items())), m_downloads + r_downloads, m_bytes + r_bytes + # ===================================================================================================== @@ -837,6 +906,7 @@ def cache_best_references(pipeline_context, header, ignore_cache=False, reftypes local_paths = cache_references(pipeline_context, best_refs, ignore_cache) return local_paths + def cache_references(pipeline_context, bestrefs, ignore_cache=False): """Given a CRDS context `pipeline_context` and `bestrefs` dictionary, download missing reference files and cache them on the local file system. @@ -856,6 +926,7 @@ def cache_references(pipeline_context, bestrefs, ignore_cache=False): return refs + def _get_cache_filelist_and_report_errors(bestrefs): """Compute the list of files to download based on the `bestrefs` dictionary, skimming off and reporting errors, and raising an exception on the last error seen. @@ -891,6 +962,7 @@ def _get_cache_filelist_and_report_errors(bestrefs): raise last_error return wanted + def _squash_unicode_in_bestrefs(bestrefs, localrefs): """Given bestrefs dictionariesy `bestrefs` and `localrefs`, make sure there are no unicode strings anywhere in the keys or complex @@ -901,7 +973,7 @@ def _squash_unicode_in_bestrefs(bestrefs, localrefs): if isinstance(refname, tuple): refs[str(filetype)] = tuple([str(localrefs[name]) for name in refname]) elif isinstance(refname, dict): - refs[str(filetype)] = { name : str(localrefs[name]) for name in refname } + refs[str(filetype)] = {name: str(localrefs[name]) for name in refname} elif isinstance(refname, str): if "NOT FOUND" in refname: refs[str(filetype)] = str(refname) @@ -912,6 +984,7 @@ def _squash_unicode_in_bestrefs(bestrefs, localrefs): "Unhandled bestrefs return value type for", srepr(filetype)) return refs + # ===================================================================================================== # These functions are deprecated and only work when the full CRDS library is installed, and only for @@ -926,11 +999,12 @@ def cache_best_references_for_dataset(pipeline_context, dataset, header = get_minimum_header(pipeline_context, dataset, ignore_cache) return cache_best_references(pipeline_context, header, ignore_cache) + def get_minimum_header(context, dataset, ignore_cache=False): """Given a `dataset` and a `context`, extract relevant header information from the `dataset`. """ import crds dump_mappings(context, ignore_cache=ignore_cache) - ctx = crds.get_pickled_mapping(context) # reviewed + ctx = crds.get_pickled_mapping(context) # reviewed return ctx.get_minimum_header(dataset) From 4eb0ae085406263dd5b43500b95691e0423bb4af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=20Ke=C3=AFn?= <3181182+alphasentaurii@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:04:16 -0400 Subject: [PATCH 13/22] api/replace-ops-with-latest (#1067) * revert back to v1 zendesk action * replace with latest * handle latest and build as context, flexibility for allowing operational * better checks for latest,build in jsonapi related calls * typo - should be = * fix typo --- crds/client/api.py | 51 ++++++++++++++++++++++++++++++--------- crds/core/cmdline.py | 7 +++--- crds/core/config.py | 21 +++++++++++----- crds/core/generic_tpn.py | 5 +++- crds/core/heavy_client.py | 36 ++++++++++++++++++--------- crds/list.py | 29 +++++++++++++++++----- crds/submit/submit.py | 3 ++- crds/sync.py | 26 ++++++++++++++------ 8 files changed, 131 insertions(+), 47 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 9359f38af..8d3a753e0 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -34,6 +34,7 @@ __all__ = [ "get_default_observatory", "get_default_context", + "get_build_context", "get_context_by_date", "get_server_info", "get_cached_server_info", # deprecated @@ -337,16 +338,37 @@ def get_aui_best_references(date, dataset_ids): @utils.cached -def get_default_context(observatory=None, state="operational"): +def get_default_context(observatory=None, state="latest"): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ return str(S.get_default_context(observatory, state)) +def get_build_context(observatory=None): + """If available, return the name of the build context pipeline mapping in use for processing + files for `observatory`. Initially only planned use is for jwst but other mission + calibration pipeline sw is included as a template. If no match found, returns latest + (formerly operational) context. + """ + if observatory == 'jwst': + calver = get_jwst_cal() + else: + cal = dict(roman='romancal', hst='caldp') + try: + calver = importlib.metadata.version(cal.get(observatory,'')) + except importlib.metadata.PackageNotFoundError: + calver = '' + if calver: + calver = config.simplify_version(calver) + return str(S.get_build_context(observatory, calver)) + else: + return get_default_context(observatory=observatory, state='latest') + + @utils.cached def get_context_by_date(date, observatory=None): - """Return the name of the first operational context which precedes `date`.""" + """Return the name of the first latest context which precedes `date`.""" return str(S.get_context_by_date(date, observatory)) @@ -354,7 +376,7 @@ def get_context_by_date(date, observatory=None): def get_server_info(): """Return a dictionary of critical parameters about the server such as: - operational_context - the context in use in the operational pipeline + latest_context - the latest context in use on the server edit_context - the context which was last edited, not necessarily archived or operational yet. @@ -496,22 +518,29 @@ def get_context_history(observatory): def push_remote_context(observatory, kind, key, context): - """Upload the specified `context` of type `kind` (e.g. "operational") to the - server, informing the server of the actual configuration of the local cache - for critical systems like pipelines, not average users. This lets the server - display actual versus commanded (Set Context) operational contexts for a pipeline. + """Upload the specified `context` of type `kind` (e.g. "latest") to the + server, informing the server of the actual configuration of the local cache. + This lets the server display actual versus commanded (Set Context) latest/operational contexts. """ try: return S.push_remote_context(observatory, kind, key, context) except Exception as exc: - raise CrdsRemoteContextError( - "Server error setting pipeline context", - (observatory, kind, key, context)) from exc + if kind == 'operational': + try: + return S.push_remote_context(observatory, 'latest', key, context) + except Exception as exc: + raise CrdsRemoteContextError( + "Server error setting latest context", + (observatory, 'latest', key, context)) from exc + else: + raise CrdsRemoteContextError( + "Server error setting operational context", + (observatory, kind, key, context)) from exc def get_remote_context(observatory, pipeline_name): """Get the name of the default context last pushed from `pipeline_name` and - presumed to be operational. + presumed to be latest. """ try: return S.get_remote_context(observatory, pipeline_name) diff --git a/crds/core/cmdline.py b/crds/core/cmdline.py index de29ceab4..ac5777a0b 100644 --- a/crds/core/cmdline.py +++ b/crds/core/cmdline.py @@ -342,8 +342,9 @@ def bad_files(self): @property def default_context(self): - """Return the default operational .pmap defined by the CRDS server or cache.""" - return self.server_info["operational_context"] + """Return the default latest .pmap defined by the CRDS server or cache.""" + default = self.server_info.get("latest_context", "operational_context") + return self.server_info[default] def get_words(self, word_list): """Process a file list, expanding @-files into corresponding lists of @@ -467,7 +468,7 @@ def run(self, *args, **keys): def resolve_context(self, context): """Resolve context spec `context` into a .pmap, .imap, or .rmap filename, interpreting - date based specifications against the CRDS server operational context history. + date based specifications against the CRDS server latest context history. """ if isinstance(context, str) and context.lower() == "none": return None diff --git a/crds/core/config.py b/crds/core/config.py index ddf518510..e4efa7a0e 100644 --- a/crds/core/config.py +++ b/crds/core/config.py @@ -1211,7 +1211,7 @@ def is_cdbs_name(name): r"(?P" + r"(" + CONTEXT_DATETIME_RE_STR + r")" + r"|" + - r"(" + "edit|operational|versions" + r")" + + r"(" + "edit|operational|latest|build|versions" + r")" + r")" + r")" + r")" + @@ -1231,7 +1231,7 @@ def is_cdbs_name(name): r"(" + r"(?P" + CONTEXT_DATETIME_RE_STR + r")" + "|" + - r"(?P" + "edit|operational|versions" + r")" + + r"(?P" + "edit|operational|latest|build|versions" + r")" + r")" + r")" + r")" @@ -1250,7 +1250,7 @@ def is_cdbs_name(name): def is_mapping(mapping): """Return True IFF `mapping` has an extension indicating a CRDS mapping file.""" - return isinstance(mapping, str) and (mapping.endswith((".pmap", ".imap", ".rmap")) or mapping.lower() in ["latest", "build"]) + return isinstance(mapping, str) and mapping.endswith((".pmap", ".imap", ".rmap")) def is_simple_crds_mapping(mapping): """Return True IFF `mapping` implicitly or explicitly exists in the CRDS cache. @@ -1348,7 +1348,16 @@ def is_mapping_spec(mapping): >>> is_mapping_spec("hst-cos-deadtab-edit") True - >>> is_mapping_spec("jwst-operational") + >>> is_mapping_spec("jwst-latest") + True + + >>> is_mapping_spec("latest") + True + + >>> is_mapping_spec("jwst-build") + True + + >>> is_mapping_spec("build") True >>> is_mapping_spec("hst-foo") @@ -1357,11 +1366,11 @@ def is_mapping_spec(mapping): >>> is_mapping_spec("hst_wfc3_0001.imap") True """ - return is_mapping(mapping) or (isinstance(mapping, str) and bool(CONTEXT_RE.match(mapping))) + return is_mapping(mapping) or (isinstance(mapping, str) and bool(CONTEXT_RE.match(mapping))) or mapping in ["latest", "build"] def is_context(mapping): """Return True IFF `mapping` has an extension indicating a CRDS CONTEXT, i.e. .pmap.""" - return isinstance(mapping, str) and (mapping.endswith((".pmap",)) or mapping in ["latest", "build"]) + return isinstance(mapping, str) and mapping.endswith((".pmap",)) def is_context_spec(mapping): """Return True IFF `mapping` is a mapping name *or* a date based mapping specification. diff --git a/crds/core/generic_tpn.py b/crds/core/generic_tpn.py index 09e611c2c..af52855d4 100644 --- a/crds/core/generic_tpn.py +++ b/crds/core/generic_tpn.py @@ -292,7 +292,10 @@ def load_all_type_constraints(observatory): to customize the more generalized constraints loaded later. """ from crds.core import rmap, heavy_client - pmap_name = heavy_client.load_server_info(observatory).operational_context + try: + pmap_name = heavy_client.load_server_info(observatory).latest_context + except AttributeError: + pmap_name = heavy_client.load_server_info(observatory).operational_context pmap = rmap.get_cached_mapping(pmap_name) locator = utils.get_locator_module(observatory) for instr in pmap.selections: diff --git a/crds/core/heavy_client.py b/crds/core/heavy_client.py index 37f5c95a2..6beee8440 100644 --- a/crds/core/heavy_client.py +++ b/crds/core/heavy_client.py @@ -404,7 +404,7 @@ def get_context_name(observatory, context=None): """Return the .pmap name of the default context based on: 1. literal definitiion in `context` (jwst_0001.pmap) - 2. symbolic definition in `context` (jwst-operational or jwst-edit) + 2. symbolic definition in `context` (latest or jwst-edit) 3. date-based definition in `context` (jwst-2017-01-15T00:05:00) 4. CRDS_CONTEXT env var override @@ -415,29 +415,42 @@ def get_context_name(observatory, context=None): def get_final_context(info, context): """Based on env CRDS_CONTEXT, the `context` parameter, and the server's reported, - cached, or defaulted `operational_context`, choose the pipeline mapping which + cached, or defaulted `latest_context`, choose the pipeline mapping which defines the reference selection rules. Returns a .pmap name """ env_context = config.get_crds_env_context() - if context: # context parameter trumps all, -operational is default + try: + latest_context = str(info.latest_context) + except AttributeError: + latest_context = str(info.operational_context) + if context: # context parameter trumps all, -latest is default if context == 'latest': - input_context = str(info.operational_context) + input_context = latest_context + elif context == 'build': + input_context = api.get_build_context(info.observatory) else: input_context = context log.verbose("Using reference file selection rules", srepr(input_context), "defined by caller.") info.status = "context parameter" elif env_context: - if env_context == 'latest': - input_context = str(info.operational_context) + if env_context in ['latest', f"{info.observatory}-operational", f"{info.observatory}-latest"]: + input_context = latest_context + elif env_context in ['build', f"{info.observatory}-build"]: + input_context = api.get_build_context(info.observatory) else: input_context = env_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by environment CRDS_CONTEXT.") info.status = "env var CRDS_CONTEXT" else: - input_context = str(info.operational_context) + # Default for JWST if no env context and no explicit context is BUILD CONTEXT for installed jwst version + if info.observatory == 'jwst': + input_context = api.get_build_context('jwst') + # For other missions, default is latest (formerly operational) context + else: + input_context = latest_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by", info.status + ".") final_context = translate_date_based_context(input_context, info.observatory) return final_context @@ -459,15 +472,14 @@ def translate_date_based_context(context, observatory=None): info = get_config_info(observatory) - if context == info.observatory + "-operational" or context == "latest": - return info["operational_context"] + latest_context_names = [info.observatory + "-operational", info.observatory + "-latest", "latest"] + + if context in latest_context_names: + return info.get('latest_context', info['operational_context']) elif context == info.observatory + "-edit": return info["edit_context"] elif context == info.observatory + "-versions": return info["versions_context"] - elif context == "build": - #TODO - pass if not info.connected: raise CrdsError("Specified CRDS context by date '{}' and CRDS server is not reachable.".format(context)) diff --git a/crds/list.py b/crds/list.py index 5ac294981..0fb3c59b4 100644 --- a/crds/list.py +++ b/crds/list.py @@ -348,7 +348,10 @@ def add_args(self): self.add_argument("--tpns", nargs="*", dest="tpns", metavar="FILES", default=None, help="print the certify constraints (.tpn's) associated with the specified or implied files.") - + self.add_argument("--latest-context", action="store_true", dest="latest_context", + help="print the name of the latest context on the central CRDS server.") + self.add_argument("--build-context", action="store_true", dest="build_context", + help="print the name of the build context on the central CRDS server.") self.add_argument("--operational-context", action="store_true", dest="operational_context", help="print the name of the operational context on the central CRDS server.") self.add_argument("--remote-context", type=str, metavar="PIPELINE", @@ -375,12 +378,15 @@ def main(self): if self.args.file_properties is not None: # including [] return self.list_file_properties() - if self.args.operational_context: + if self.args.operational_context or self.args.latest_context: print(self.default_context) return if self.args.remote_context: print(self.remote_context) return + if self.args.build_context: + print(self.build_context) + return if self.args.resolve_contexts: self.list_resolved_contexts() @@ -410,7 +416,7 @@ def main(self): self.list_required_parkeys() return log.errors() - + def list_resolved_contexts(self): """Print out the literal interpretation of the contexts implied by the script's context specifiers. @@ -428,6 +434,12 @@ def remote_context(self): with log.error_on_exception("Failed resolving remote context"): return api.get_remote_context(self.observatory, self.args.remote_context) + @property + def build_context(self): + self.require_server_connection() + with log.error_on_exception("Failed resolving build context"): + return api.get_build_context(self.observatory) + @property def implied_files(self): """Return the filenames implied by a combination of the specified contexts @@ -656,9 +668,14 @@ def list_config(self): "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) - _print_dict("CRDS Server Info", server, - ["observatory", "status", "connected", "operational_context", "last_synced", - "reference_url", "mapping_url", "effective_mode"]) + try: + _print_dict("CRDS Server Info", server, + ["observatory", "status", "connected", "latest_context", "last_synced", + "reference_url", "mapping_url", "effective_mode"]) + except Exception: + _print_dict("CRDS Server Info", server, + ["observatory", "status", "connected", "operational_context", "last_synced", + "reference_url", "mapping_url", "effective_mode"]) download_uris = dict( config_uri = os.path.dirname(api.get_flex_uri("server_config")), pickle_uri = os.path.dirname(api.get_flex_uri("xyz.pmap.pkl")), diff --git a/crds/submit/submit.py b/crds/submit/submit.py index 26bdb9d9f..f317b22bc 100644 --- a/crds/submit/submit.py +++ b/crds/submit/submit.py @@ -106,7 +106,8 @@ def finish_parameters(self): locked_instrument=self.instrument, username=self.username, password=password, base_url=self.base_url) if self.args.derive_from_context is None: self.args.derive_from_context = self.observatory + "-edit" - if self.args.derive_from_context.endswith(("-edit", "-operational")): + self.args.derive_from_context.replace("-operational", "-latest") + if self.args.derive_from_context.endswith(("-edit", "-latest")): # this actually floats for concurrent deliveries self.pmap_mode = "pmap_" + self.args.derive_from_context.split("-")[-1] else: diff --git a/crds/sync.py b/crds/sync.py index 2be0d8706..c1d121ce7 100644 --- a/crds/sync.py +++ b/crds/sync.py @@ -401,7 +401,7 @@ def get_synced_references(self): return active_references def update_context(self): - """Update the CRDS operational context in the cache. Handle pipeline-specific + """Update the CRDS latest context in the cache. Handle pipeline-specific targeted features of (a) verifying a context switch as actually recorded in the local CRDS cache and (b) echoing/pushing the pipeline context back up to the CRDS server for tracking using an id/authorization key. @@ -410,7 +410,10 @@ def update_context(self): """ if not log.errors() or self.args.force_config_update: if self.args.verify_context_change: - old_context = heavy_client.load_server_info(self.observatory).operational_context + try: + old_context = heavy_client.load_server_info(self.observatory).latest_context + except AttributeError: + old_context = heavy_client.load_server_info(self.observatory).operational_context heavy_client.update_config_info(self.observatory) if self.args.verify_context_change: self.verify_context_change(old_context) @@ -444,7 +447,10 @@ def server_info(self): def verify_context_change(self, old_context): """Verify that the starting and post-sync contexts are different, or issue an error.""" - new_context = heavy_client.load_server_info(self.observatory).operational_context + try: + new_context = heavy_client.load_server_info(self.observatory).latest_context + except AttributeError: + new_context = heavy_client.load_server_info(self.observatory).operational_context if old_context == new_context: log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context)) else: @@ -452,12 +458,18 @@ def verify_context_change(self, old_context): def push_context(self): """Push the final context recorded in the local cache to the CRDS server so it can be displayed - as the operational state of a pipeline. + as the latest state of a pipeline. """ info = heavy_client.load_server_info(self.observatory) - with log.error_on_exception("Failed pushing cached operational context name to CRDS server"): - api.push_remote_context(self.observatory, "operational", self.args.push_context, info.operational_context) - log.info("Pushed cached operational context name", repr(info.operational_context), "to CRDS server") + try: + ctx = 'latest' + latest_context = info.latest_context + except AttributeError: + ctx = 'operational' + latest_context = info.operational_context + with log.error_on_exception(f"Failed pushing cached {ctx} context name to CRDS server"): + api.push_remote_context(self.observatory, ctx, self.args.push_context, latest_context) + log.info(f"Pushed cached {ctx} context name", repr(latest_context), "to CRDS server") # ------------------------------------------------------------------------------------------ From 2ce2c6eaf96567e3f4f4364b8ea6f449483e679b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ru=20Ke=C3=AFn?= <3181182+alphasentaurii@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:50:05 -0400 Subject: [PATCH 14/22] buildcontext/fallback (#1070) * revert back to v1 zendesk action * if no cal software installed, default to latest --- crds/client/api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 8d3a753e0..440283797 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -351,18 +351,20 @@ def get_build_context(observatory=None): calibration pipeline sw is included as a template. If no match found, returns latest (formerly operational) context. """ - if observatory == 'jwst': - calver = get_jwst_cal() - else: - cal = dict(roman='romancal', hst='caldp') - try: + try: + if observatory == 'jwst': + calver = get_jwst_cal() + else: + cal = dict(roman='romancal', hst='caldp') calver = importlib.metadata.version(cal.get(observatory,'')) - except importlib.metadata.PackageNotFoundError: - calver = '' + except importlib.metadata.PackageNotFoundError: + calver = '' + log.warning("Cal SW not found, defaulting to latest.") if calver: calver = config.simplify_version(calver) return str(S.get_build_context(observatory, calver)) else: + return get_default_context(observatory=observatory, state='latest') From 89f92102f710e323d9bc226cfb5c29f96360d3ff Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Mon, 16 Sep 2024 16:49:16 -0400 Subject: [PATCH 15/22] generalize calver lookup, handle various cases --- crds/client/api.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 440283797..1343f831d 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -240,9 +240,17 @@ def _get_file_info_map(observatory, files, fields): return infos -def get_jwst_cal(): - """Return the version of jwst code.""" - cal_version = importlib.metadata.version('jwst') +def get_cal_version(observatory): + """Return the version of observatory calibration software.""" + cal_version = '' + if observatory: + cal = dict(jwst='jwst', roman='romancal', hst='caldp')[observatory] + try: + cal_version = importlib.metadata.version(cal) + calver = config.simplify_version(calver) + log.info(f"Calibration SW Found: {cal} {calver}") + except importlib.metadata.PackageNotFoundError: + log.warning("Calibration SW not found, defaulting to latest.") return cal_version @@ -342,6 +350,8 @@ def get_default_context(observatory=None, state="latest"): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ + if state == "build": + return get_build_context(observatory=observatory) return str(S.get_default_context(observatory, state)) @@ -351,20 +361,15 @@ def get_build_context(observatory=None): calibration pipeline sw is included as a template. If no match found, returns latest (formerly operational) context. """ - try: - if observatory == 'jwst': - calver = get_jwst_cal() - else: - cal = dict(roman='romancal', hst='caldp') - calver = importlib.metadata.version(cal.get(observatory,'')) - except importlib.metadata.PackageNotFoundError: - calver = '' - log.warning("Cal SW not found, defaulting to latest.") + if observatory is None: + try: + observatory = str(S).split("=")[1].split("-")[0].split("/")[-1] + except ServiceError: + observatory = os.environ.get('CRDS_SERVER_URL', '').split("-")[0].split("/")[-1] + calver = get_cal_version(observatory) if calver: - calver = config.simplify_version(calver) return str(S.get_build_context(observatory, calver)) else: - return get_default_context(observatory=observatory, state='latest') From 6b484a81b7db5cfd3b3830e7c41623ee33e6bb8d Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Mon, 16 Sep 2024 16:59:58 -0400 Subject: [PATCH 16/22] fix typo cal_version not calver --- crds/client/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 1343f831d..f7c4241d3 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -247,8 +247,8 @@ def get_cal_version(observatory): cal = dict(jwst='jwst', roman='romancal', hst='caldp')[observatory] try: cal_version = importlib.metadata.version(cal) - calver = config.simplify_version(calver) - log.info(f"Calibration SW Found: {cal} {calver}") + cal_version = config.simplify_version(cal_version) + log.info(f"Calibration SW Found: {cal} {cal_version}") except importlib.metadata.PackageNotFoundError: log.warning("Calibration SW not found, defaulting to latest.") return cal_version From 5537c4dcc2f9c7d331de53a1a689bff0ad98bcc7 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Thu, 19 Sep 2024 11:24:26 -0400 Subject: [PATCH 17/22] ensure get_default_context with no args defaults to build for jwst, else latest --- crds/client/api.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index f7c4241d3..98dffd56e 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -346,11 +346,12 @@ def get_aui_best_references(date, dataset_ids): @utils.cached -def get_default_context(observatory=None, state="latest"): +def get_default_context(observatory=None, state=None): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ - if state == "build": + observatory = get_default_observatory() if observatory is None else observatory + if state == "build" or (observatory == "jwst" and state not in ["edit", "latest"]): return get_build_context(observatory=observatory) return str(S.get_default_context(observatory, state)) @@ -358,19 +359,18 @@ def get_default_context(observatory=None, state="latest"): def get_build_context(observatory=None): """If available, return the name of the build context pipeline mapping in use for processing files for `observatory`. Initially only planned use is for jwst but other mission - calibration pipeline sw is included as a template. If no match found, returns latest - (formerly operational) context. + calibration pipeline sw is included as a template. If exact match is not found, an attempt to + find next closest (previous) patch version is made. Ultimate fallback is to the latest + (formerly 'operational') context. """ - if observatory is None: - try: - observatory = str(S).split("=")[1].split("-")[0].split("/")[-1] - except ServiceError: - observatory = os.environ.get('CRDS_SERVER_URL', '').split("-")[0].split("/")[-1] + observatory = get_default_observatory() if observatory is None else observatory calver = get_cal_version(observatory) if calver: - return str(S.get_build_context(observatory, calver)) - else: - return get_default_context(observatory=observatory, state='latest') + try: + return str(S.get_build_context(observatory, calver)) + except ServiceError: + log.warning("Server build context could not be identified. Using 'latest' instead.") + return get_default_context(observatory, "latest") @utils.cached @@ -615,7 +615,7 @@ def get_default_observatory(): 4. jwst """ obs = config.OBSERVATORY.get() - if obs != "none": + if obs not in ["none", ""]: return obs return observatory_from_string(get_crds_server()) or \ get_server_observatory() or \ From e7c970438db4fbbc9cc929426f6d00a06020d32f Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Thu, 19 Sep 2024 11:28:00 -0400 Subject: [PATCH 18/22] update changelog --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 63f48bc7c..c1aa95c14 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,10 +4,13 @@ General ------- -- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. +- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1075] - Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] +- `client.api.get_default_context` by default returns build context for jwst, else latest. This can still be overridden by explicitly passing a value into optional arg `state`. [#1069] + + 11.18.4 (2024-09-10) ==================== From 898f9d1ae005b155b6f6d5e4ea05b4bef9f05d2b Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Thu, 19 Sep 2024 11:56:40 -0400 Subject: [PATCH 19/22] handle env var CRDS_OBSERVATORY empty string --- crds/client/api.py | 2 +- crds/core/cmdline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crds/client/api.py b/crds/client/api.py index 98dffd56e..722ca4c17 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -615,7 +615,7 @@ def get_default_observatory(): 4. jwst """ obs = config.OBSERVATORY.get() - if obs not in ["none", ""]: + if obs not in ["none", "", None]: return obs return observatory_from_string(get_crds_server()) or \ get_server_observatory() or \ diff --git a/crds/core/cmdline.py b/crds/core/cmdline.py index ac5777a0b..19de075b1 100644 --- a/crds/core/cmdline.py +++ b/crds/core/cmdline.py @@ -222,7 +222,7 @@ def observatory(self): return self.set_server("roman") obs = config.OBSERVATORY.get() - if obs != "none": + if obs not in ["none", "", None]: return self.set_server(obs.lower()) url = os.environ.get("CRDS_SERVER_URL", None) From 58bc6b9264ecf41b922e6e48c25690483094a5d3 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Wed, 25 Sep 2024 12:26:31 -0400 Subject: [PATCH 20/22] remove merge conflict in changelog --- CHANGES.rst | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c1aa95c14..50c4b9dae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,16 +1,3 @@ -12.0.0 (unreleased) -==================== - -General -------- - -- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1075] - -- Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] - -- `client.api.get_default_context` by default returns build context for jwst, else latest. This can still be overridden by explicitly passing a value into optional arg `state`. [#1069] - - 11.18.4 (2024-09-10) ==================== From 16ff4f0b5a593135598beee68816ba6a3b43d827 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Wed, 25 Sep 2024 12:27:56 -0400 Subject: [PATCH 21/22] update changelog --- CHANGES.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 50c4b9dae..c1aa95c14 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,16 @@ +12.0.0 (unreleased) +==================== + +General +------- + +- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1075] + +- Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] + +- `client.api.get_default_context` by default returns build context for jwst, else latest. This can still be overridden by explicitly passing a value into optional arg `state`. [#1069] + + 11.18.4 (2024-09-10) ==================== From 9b061de002ee449ff7401d85de487b73943c6d73 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Wed, 25 Sep 2024 12:30:04 -0400 Subject: [PATCH 22/22] increment pr --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c1aa95c14..26f216fc3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ General ------- -- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1075] +- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1076] - Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062]