diff --git a/CHANGES.rst b/CHANGES.rst index 50c4b9dae..403265328 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,16 @@ +12.0.0 (unreleased) +==================== + +General +------- + +- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1076] + +- Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] + +- `client.api.get_default_context` by default returns build context for jwst, else latest. This can still be overridden by explicitly passing a value into optional arg `state`. [#1069] + + 11.18.4 (2024-09-10) ==================== @@ -7,6 +20,7 @@ General - Replaced deprecated SafeConfigParser with ConfigParser in crds.core.config [#1065] - moved DMS requirement correlations with tests from ``@metrics_logger`` test decorators to ``test/dms_requirement_tests.json`` [#1064] + 11.18.3 (2024-09-03) ==================== diff --git a/crds/client/api.py b/crds/client/api.py index 8d5e7c059..722ca4c17 100644 --- a/crds/client/api.py +++ b/crds/client/api.py @@ -8,6 +8,7 @@ import re import zlib import html +import importlib.metadata from urllib import request import warnings import json @@ -33,6 +34,7 @@ __all__ = [ "get_default_observatory", "get_default_context", + "get_build_context", "get_context_by_date", "get_server_info", "get_cached_server_info", # deprecated @@ -44,7 +46,7 @@ "list_mappings", "list_references", - "get_url", # deprecated + "get_url", # deprecated "get_flex_uri", "get_file_info", "get_file_info_map", @@ -78,7 +80,7 @@ "jpoll_abort", "get_system_versions", - ] +] # ============================================================================ @@ -86,7 +88,8 @@ URL_SUFFIX = "/json/" -S = None # Proxy server +S = None # Proxy server + def set_crds_server(url): """Configure the CRDS JSON services server to `url`, @@ -100,6 +103,7 @@ def set_crds_server(url): URL = url + URL_SUFFIX S = CheckingProxy(URL, version="1.0") + def get_crds_server(): """Return the base URL for the CRDS JSON RPC server. """ @@ -108,8 +112,10 @@ def get_crds_server(): log.warning("CRDS_SERVER_URL does not start with https:// ::", url) return url + # ============================================================================= + @utils.cached def list_mappings(observatory=None, glob_pattern="*"): """Return the list of mappings associated with `observatory` @@ -117,6 +123,7 @@ def list_mappings(observatory=None, glob_pattern="*"): """ return [str(x) for x in S.list_mappings(observatory, glob_pattern)] + @utils.cached def list_references(observatory=None, glob_pattern="*"): """Return the list of references associated with `observatory` @@ -124,6 +131,7 @@ def list_references(observatory=None, glob_pattern="*"): """ return [str(x) for x in S.list_references(observatory, glob_pattern)] + def get_mapping_url(pipeline_context, mapping): """Returns a URL for the specified pmap, imap, or rmap file. DEPRECATED """ @@ -131,6 +139,7 @@ def get_mapping_url(pipeline_context, mapping): "crds.client.get_mapping_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_mapping_url(pipeline_context, mapping) + def is_known_mapping(mapping): """Return True iff `mapping` is a known/official CRDS mapping file.""" try: @@ -138,6 +147,7 @@ def is_known_mapping(mapping): except ServiceError: return False + @utils.cached def get_mapping_names(pipeline_context): """Get the complete set of pmap, imap, and rmap basenames required @@ -146,6 +156,7 @@ def get_mapping_names(pipeline_context): """ return [str(x) for x in S.get_mapping_names(pipeline_context)] + def get_reference_url(pipeline_context, reference): """Returns a URL for the specified reference file. DEPRECATED """ @@ -153,6 +164,7 @@ def get_reference_url(pipeline_context, reference): "crds.client.get_reference_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_reference_url(pipeline_context, reference) + def get_url(pipeline_context, filename): """Return the URL for a CRDS reference or mapping file. DEPRECATED """ @@ -160,6 +172,7 @@ def get_url(pipeline_context, filename): "crds.client.get_url()", "2020-09-01", "crds.client.get_flex_uri()") return S.get_url(pipeline_context, filename) + def get_flex_uri(filename, observatory=None): """If environment variables define the base URI for `filename`, append filename and return the combined URI. @@ -192,20 +205,23 @@ def get_flex_uri(filename, observatory=None): uri += filename return uri + def _unpack_info(info, section, observatory): """Return info[section][observatory] if info[section] is defined. Otherwise return "none". """ sect = info.get(section) if sect: - return sect[observatory] + return sect[observatory] else: return "none" + def get_file_info(pipeline_context, filename): """Return a dictionary of CRDS information about `filename`.""" return S.get_file_info(pipeline_context, filename) + def get_file_info_map(observatory, files=None, fields=None): """Return the info { filename : { info } } on `files` of `observatory`. `fields` can be used to limit info returned to specified keys. @@ -216,12 +232,28 @@ def get_file_info_map(observatory, files=None, fields=None): fields = tuple(sorted(fields)) return _get_file_info_map(observatory, files, fields) + @utils.cached def _get_file_info_map(observatory, files, fields): """Memory cached version of get_file_info_map() service.""" infos = S.get_file_info_map(observatory, files, fields) return infos + +def get_cal_version(observatory): + """Return the version of observatory calibration software.""" + cal_version = '' + if observatory: + cal = dict(jwst='jwst', roman='romancal', hst='caldp')[observatory] + try: + cal_version = importlib.metadata.version(cal) + cal_version = config.simplify_version(cal_version) + log.info(f"Calibration SW Found: {cal} {cal_version}") + except importlib.metadata.PackageNotFoundError: + log.warning("Calibration SW not found, defaulting to latest.") + return cal_version + + def get_total_bytes(info_map): """Return the total byte count of file info map `info_map`.""" try: @@ -230,6 +262,7 @@ def get_total_bytes(info_map): log.error("Error computing total byte count: ", str(exc)) return -1 + def get_sqlite_db(observatory): """Download the CRDS database as a SQLite database.""" assert not config.get_cache_readonly(), "Readonly cache, updating the SQLite database cannot be done." @@ -241,6 +274,7 @@ def get_sqlite_db(observatory): db_out.write(data) return path + @utils.cached def get_reference_names(pipeline_context): """Get the complete set of reference file basenames required @@ -248,6 +282,7 @@ def get_reference_names(pipeline_context): """ return [str(x) for x in S.get_reference_names(pipeline_context)] + def get_best_references(pipeline_context, header, reftypes=None): """Get best references for dict-like `header` relative to `pipeline_context`. @@ -261,13 +296,14 @@ def get_best_references(pipeline_context, header, reftypes=None): Raises CrdsLookupError, typically for problems with header values """ - header = { str(key):str(value) for (key,value) in header.items() } + header = {str(key): str(value) for (key, value) in header.items()} try: bestrefs = S.get_best_references(pipeline_context, dict(header), reftypes) except Exception as exc: raise CrdsLookupError(str(exc)) from exc return bestrefs + def get_best_references_by_ids(context, dataset_ids, reftypes=None, include_headers=False): """Get best references for the specified `dataset_ids` and reference types. If reftypes is None, all types are returned. @@ -280,6 +316,7 @@ def get_best_references_by_ids(context, dataset_ids, reftypes=None, include_head raise CrdsLookupError(str(exc)) from exc return bestrefs + def get_best_references_by_header_map(context, header_map, reftypes=None): """Get best references for header_map = { dataset_id : header, ...}, } and reference types where a header is a dictionary of matching parameters. @@ -307,23 +344,46 @@ def get_aui_best_references(date, dataset_ids): raise CrdsLookupError(str(exc)) from exc return bestrefs_map + @utils.cached -def get_default_context(observatory=None): +def get_default_context(observatory=None, state=None): """Return the name of the latest pipeline mapping in use for processing files for `observatory`. """ - return str(S.get_default_context(observatory)) + observatory = get_default_observatory() if observatory is None else observatory + if state == "build" or (observatory == "jwst" and state not in ["edit", "latest"]): + return get_build_context(observatory=observatory) + return str(S.get_default_context(observatory, state)) + + +def get_build_context(observatory=None): + """If available, return the name of the build context pipeline mapping in use for processing + files for `observatory`. Initially only planned use is for jwst but other mission + calibration pipeline sw is included as a template. If exact match is not found, an attempt to + find next closest (previous) patch version is made. Ultimate fallback is to the latest + (formerly 'operational') context. + """ + observatory = get_default_observatory() if observatory is None else observatory + calver = get_cal_version(observatory) + if calver: + try: + return str(S.get_build_context(observatory, calver)) + except ServiceError: + log.warning("Server build context could not be identified. Using 'latest' instead.") + return get_default_context(observatory, "latest") + @utils.cached def get_context_by_date(date, observatory=None): - """Return the name of the first operational context which precedes `date`.""" + """Return the name of the first latest context which precedes `date`.""" return str(S.get_context_by_date(date, observatory)) + @utils.cached def get_server_info(): """Return a dictionary of critical parameters about the server such as: - operational_context - the context in use in the operational pipeline + latest_context - the latest context in use on the server edit_context - the context which was last edited, not necessarily archived or operational yet. @@ -361,12 +421,14 @@ def get_server_info(): info["download_metadata"] = proxy.crds_encode(metadata) return info + @utils.cached def get_download_metadata(): "Defer and cache decoding of download_metadata field of server info.""" info = get_server_info() return proxy.crds_decode(info["download_metadata"]) + def _get_server_info(): """Fetch the server info dict. If CRDS_CONFIG_URI is set then download that URL and load json from the contents. Otherwise, @@ -394,23 +456,28 @@ def _get_server_info(): srepr(exc)) from exc return info + get_cached_server_info = get_server_info + def get_server_version(): """Return the API version of the current CRDS server.""" info = get_server_info() return info["crds_version"]["str"] + def get_dataset_headers_by_id(context, dataset_ids, datasets_since=None): """Return { dataset_id : { header } } for `dataset_ids`.""" context = os.path.basename(context) return S.get_dataset_headers_by_id(context, dataset_ids, datasets_since) + def get_dataset_ids(context, instrument, datasets_since=None): """Return [ dataset_id, ...] for `instrument`.""" context = os.path.basename(context) return S.get_dataset_ids(context, instrument, datasets_since) + @utils.cached def get_required_parkeys(context): """Return a mapping from instruments to lists of parameter names required to @@ -421,12 +488,14 @@ def get_required_parkeys(context): context = os.path.basename(context) return S.get_required_parkeys(context) + def get_dataset_headers_by_instrument(context, instrument, datasets_since=None): """return { dataset_id:header, ...} for every `dataset_id` for `instrument`.""" log.verbose("Dumping datasets for", repr(instrument)) ids = get_dataset_ids(context, instrument, datasets_since) return dict(get_dataset_headers_unlimited(context, ids)) + def get_dataset_headers_unlimited(context, ids): """Generate (dataset_id, header) for `ids`, potentially more `ids` than can be serviced with a single JSONRPC request. @@ -435,16 +504,18 @@ def get_dataset_headers_unlimited(context, ids): """ max_ids_per_rpc = get_server_info().get("max_headers_per_rpc", 500) for i in range(0, len(ids), max_ids_per_rpc): - log.verbose("Dumping dataset headers", i , "of", len(ids), verbosity=20) - id_slice = ids[i : i + max_ids_per_rpc] + log.verbose("Dumping dataset headers", i, "of", len(ids), verbosity=20) + id_slice = ids[i: i + max_ids_per_rpc] header_slice = get_dataset_headers_by_id(context, id_slice) for item in header_slice.items(): yield item + def get_affected_datasets(observatory, old_context=None, new_context=None): """Return a structure describing the ids affected by the last context change.""" return utils.Struct(S.get_affected_datasets(observatory, old_context, new_context)) + def get_context_history(observatory): """Fetch the history of context transitions, a list of history era tuples: @@ -452,22 +523,31 @@ def get_context_history(observatory): """ return sorted(tuple(x) for x in S.get_context_history(observatory)) + def push_remote_context(observatory, kind, key, context): - """Upload the specified `context` of type `kind` (e.g. "operational") to the - server, informing the server of the actual configuration of the local cache - for critical systems like pipelines, not average users. This lets the server - display actual versus commanded (Set Context) operational contexts for a pipeline. + """Upload the specified `context` of type `kind` (e.g. "latest") to the + server, informing the server of the actual configuration of the local cache. + This lets the server display actual versus commanded (Set Context) latest/operational contexts. """ try: return S.push_remote_context(observatory, kind, key, context) except Exception as exc: - raise CrdsRemoteContextError( - "Server error setting pipeline context", - (observatory, kind, key, context)) from exc + if kind == 'operational': + try: + return S.push_remote_context(observatory, 'latest', key, context) + except Exception as exc: + raise CrdsRemoteContextError( + "Server error setting latest context", + (observatory, 'latest', key, context)) from exc + else: + raise CrdsRemoteContextError( + "Server error setting operational context", + (observatory, kind, key, context)) from exc + def get_remote_context(observatory, pipeline_name): """Get the name of the default context last pushed from `pipeline_name` and - presumed to be operational. + presumed to be latest. """ try: return S.get_remote_context(observatory, pipeline_name) @@ -476,8 +556,10 @@ def get_remote_context(observatory, pipeline_name): "Server error resolving context in use by pipeline", (observatory, pipeline_name)) from exc + # ============================================================================== + def jpoll_pull_messages(key, since_id=None): """Return a list of jpoll json message objects from the channel associated with `key` sent after datetime string `since` or since the last pull if @@ -490,12 +572,15 @@ def jpoll_pull_messages(key, since_id=None): messages.append(decoded) return messages + def jpoll_abort(key): """Request that the process writing to jpoll terminate on its next write.""" return S.jpoll_abort(key) + # ============================================================================== + def get_system_versions(master_version, context=None): """Return the versions Struct associated with cal s/w master_version as defined by `context` which can be defined as "null", "none", or None to use @@ -503,10 +588,13 @@ def get_system_versions(master_version, context=None): """ return utils.Struct(S.get_system_versions(master_version, str(context))) + # ============================================================================== + HARD_DEFAULT_OBS = "jwst" + def get_server_observatory(): """Return the default observatory according to the server, or None.""" try: @@ -517,6 +605,7 @@ def get_server_observatory(): server_obs = observatory_from_string(pmap) return server_obs + def get_default_observatory(): """Based on the environment, cache, and server, determine the default observatory. @@ -526,11 +615,12 @@ def get_default_observatory(): 4. jwst """ obs = config.OBSERVATORY.get() - if obs != "none": + if obs not in ["none", "", None]: return obs return observatory_from_string(get_crds_server()) or \ - get_server_observatory() or \ - "jwst" + get_server_observatory() or \ + "jwst" + def observatory_from_string(string): """If an observatory name is in `string`, return it, otherwise return None.""" @@ -539,23 +629,28 @@ def observatory_from_string(string): return observatory return None + # ============================================================================== + def file_progress(activity, name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files): """Output progress information for `activity` on file `name` at `path`.""" return "{activity} {path!s:<55} {bytes} bytes ({nth_file} / {total_files} files) ({bytes_so_far} / {total_bytes} bytes)".format( activity=activity, path=path, bytes=utils.human_format_number(bytes), - nth_file=nth_file+1, + nth_file=nth_file + 1, total_files=total_files, bytes_so_far=utils.human_format_number(bytes_so_far).strip(), total_bytes=utils.human_format_number(total_bytes).strip()) + # ============================================================================== + class FileCacher: """FileCacher gets remote files with simple names into a local cache.""" + def __init__(self, pipeline_context, ignore_cache=False, raise_exceptions=True): self.pipeline_context = pipeline_context self.observatory = self.observatory_from_context() @@ -576,7 +671,7 @@ def get_local_files(self, names): names2 = names[:] for refname in names2: if re.match(r"\w+\.r[0-9]h$", refname): - names.append(refname[:-1]+"d") + names.append(refname[:-1] + "d") downloads = [] for name in names: @@ -621,7 +716,8 @@ def download_files(self, downloads, localpaths): self.info_map = {} for filename in downloads: self.info_map[filename] = download_metadata.get(filename, "NOT FOUND unknown to server") - if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), verbosity=70): + if config.writable_cache_or_verbose("Readonly cache, skipping download of (first 5):", repr(downloads[:5]), + verbosity=70): bytes_so_far = 0 total_files = len(downloads) total_bytes = get_total_bytes(self.info_map) @@ -630,7 +726,8 @@ def download_files(self, downloads, localpaths): if "NOT FOUND" in self.info_map[name]: raise CrdsDownloadError("file is not known to CRDS server.") bytes, path = self.catalog_file_size(name), localpaths[name] - log.info(file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files)) + log.info( + file_progress("Fetching", name, path, bytes, bytes_so_far, total_bytes, nth_file, total_files)) self.download(name, path) bytes_so_far += os.stat(path).st_size except Exception as exc: @@ -658,7 +755,7 @@ def download(self, name, localpath): "at CRDS server", srepr(get_crds_server()), "with mode", srepr(config.get_download_mode()), ":", str(exc)) from exc - except: # mainly for control-c, catch it and throw it. + except: # mainly for control-c, catch it and throw it. self.remove_file(localpath) raise @@ -715,7 +812,8 @@ def get_data_http(self, filename): stats.increment("bytes", len(data)) status = stats.status("bytes") bytes_so_far = " ".join(status[0].split()[:-1]) - log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], verbosity=20) + log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], + verbosity=20) yield data data = infile.read(config.CRDS_DATA_CHUNK_SIZE) except Exception as exc: @@ -725,7 +823,7 @@ def get_data_http(self, filename): finally: try: infile.close() - except UnboundLocalError: # maybe the open failed. + except UnboundLocalError: # maybe the open failed. pass def get_url(self, filename): @@ -754,8 +852,10 @@ def verify_file(self, filename, localpath): else: log.verbose("Skipping sha1sum check since server doesn't know it.") + # ============================================================================== + def dump_mappings3(pipeline_context, ignore_cache=False, mappings=None, raise_exceptions=True): """Given a `pipeline_context`, determine the closure of CRDS mappings for it and cache them on the local file system. @@ -770,6 +870,7 @@ def dump_mappings3(pipeline_context, ignore_cache=False, mappings=None, raise_ex mappings = list(reversed(sorted(set(mappings)))) return FileCacher(pipeline_context, ignore_cache, raise_exceptions).get_local_files(mappings) + def dump_mappings(*args, **keys): """See dump_mappings3. @@ -777,6 +878,7 @@ def dump_mappings(*args, **keys): """ return dump_mappings3(*args, **keys)[0] + def dump_references3(pipeline_context, baserefs=None, ignore_cache=False, raise_exceptions=True): """Given a pipeline `pipeline_context` and list of `baserefs` reference file basenames, obtain the set of reference files and cache them on the @@ -796,6 +898,7 @@ def dump_references3(pipeline_context, baserefs=None, ignore_cache=False, raise_ baserefs = sorted(set(baserefs)) return FileCacher(pipeline_context, ignore_cache, raise_exceptions).get_local_files(baserefs) + def dump_references(*args, **keys): """See dump_references3. @@ -803,6 +906,7 @@ def dump_references(*args, **keys): """ return dump_references3(*args, **keys)[0] + def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exceptions=True): """Unified interface to dump any file in `files`, mapping or reference. @@ -812,8 +916,8 @@ def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exce pipeline_context = get_default_context() if files is None: files = get_mapping_names(pipeline_context) - mappings = [ os.path.basename(name) for name in files if config.is_mapping(name) ] - references = [ os.path.basename(name) for name in files if not config.is_mapping(name) ] + mappings = [os.path.basename(name) for name in files if config.is_mapping(name)] + references = [os.path.basename(name) for name in files if not config.is_mapping(name)] if mappings: m_paths, m_downloads, m_bytes = dump_mappings3( pipeline_context, mappings=mappings, ignore_cache=ignore_cache, raise_exceptions=raise_exceptions) @@ -824,7 +928,8 @@ def dump_files(pipeline_context=None, files=None, ignore_cache=False, raise_exce pipeline_context, baserefs=references, ignore_cache=ignore_cache, raise_exceptions=raise_exceptions) else: r_paths, r_downloads, r_bytes = {}, 0, 0 - return dict(list(m_paths.items())+list(r_paths.items())), m_downloads + r_downloads, m_bytes + r_bytes + return dict(list(m_paths.items()) + list(r_paths.items())), m_downloads + r_downloads, m_bytes + r_bytes + # ===================================================================================================== @@ -837,6 +942,7 @@ def cache_best_references(pipeline_context, header, ignore_cache=False, reftypes local_paths = cache_references(pipeline_context, best_refs, ignore_cache) return local_paths + def cache_references(pipeline_context, bestrefs, ignore_cache=False): """Given a CRDS context `pipeline_context` and `bestrefs` dictionary, download missing reference files and cache them on the local file system. @@ -856,6 +962,7 @@ def cache_references(pipeline_context, bestrefs, ignore_cache=False): return refs + def _get_cache_filelist_and_report_errors(bestrefs): """Compute the list of files to download based on the `bestrefs` dictionary, skimming off and reporting errors, and raising an exception on the last error seen. @@ -891,6 +998,7 @@ def _get_cache_filelist_and_report_errors(bestrefs): raise last_error return wanted + def _squash_unicode_in_bestrefs(bestrefs, localrefs): """Given bestrefs dictionariesy `bestrefs` and `localrefs`, make sure there are no unicode strings anywhere in the keys or complex @@ -901,7 +1009,7 @@ def _squash_unicode_in_bestrefs(bestrefs, localrefs): if isinstance(refname, tuple): refs[str(filetype)] = tuple([str(localrefs[name]) for name in refname]) elif isinstance(refname, dict): - refs[str(filetype)] = { name : str(localrefs[name]) for name in refname } + refs[str(filetype)] = {name: str(localrefs[name]) for name in refname} elif isinstance(refname, str): if "NOT FOUND" in refname: refs[str(filetype)] = str(refname) @@ -912,6 +1020,7 @@ def _squash_unicode_in_bestrefs(bestrefs, localrefs): "Unhandled bestrefs return value type for", srepr(filetype)) return refs + # ===================================================================================================== # These functions are deprecated and only work when the full CRDS library is installed, and only for @@ -926,11 +1035,12 @@ def cache_best_references_for_dataset(pipeline_context, dataset, header = get_minimum_header(pipeline_context, dataset, ignore_cache) return cache_best_references(pipeline_context, header, ignore_cache) + def get_minimum_header(context, dataset, ignore_cache=False): """Given a `dataset` and a `context`, extract relevant header information from the `dataset`. """ import crds dump_mappings(context, ignore_cache=ignore_cache) - ctx = crds.get_pickled_mapping(context) # reviewed + ctx = crds.get_pickled_mapping(context) # reviewed return ctx.get_minimum_header(dataset) diff --git a/crds/core/cmdline.py b/crds/core/cmdline.py index de29ceab4..19de075b1 100644 --- a/crds/core/cmdline.py +++ b/crds/core/cmdline.py @@ -222,7 +222,7 @@ def observatory(self): return self.set_server("roman") obs = config.OBSERVATORY.get() - if obs != "none": + if obs not in ["none", "", None]: return self.set_server(obs.lower()) url = os.environ.get("CRDS_SERVER_URL", None) @@ -342,8 +342,9 @@ def bad_files(self): @property def default_context(self): - """Return the default operational .pmap defined by the CRDS server or cache.""" - return self.server_info["operational_context"] + """Return the default latest .pmap defined by the CRDS server or cache.""" + default = self.server_info.get("latest_context", "operational_context") + return self.server_info[default] def get_words(self, word_list): """Process a file list, expanding @-files into corresponding lists of @@ -467,7 +468,7 @@ def run(self, *args, **keys): def resolve_context(self, context): """Resolve context spec `context` into a .pmap, .imap, or .rmap filename, interpreting - date based specifications against the CRDS server operational context history. + date based specifications against the CRDS server latest context history. """ if isinstance(context, str) and context.lower() == "none": return None diff --git a/crds/core/config.py b/crds/core/config.py index 24e95b1e0..e4efa7a0e 100644 --- a/crds/core/config.py +++ b/crds/core/config.py @@ -1211,7 +1211,7 @@ def is_cdbs_name(name): r"(?P" + r"(" + CONTEXT_DATETIME_RE_STR + r")" + r"|" + - r"(" + "edit|operational|versions" + r")" + + r"(" + "edit|operational|latest|build|versions" + r")" + r")" + r")" + r")" + @@ -1231,7 +1231,7 @@ def is_cdbs_name(name): r"(" + r"(?P" + CONTEXT_DATETIME_RE_STR + r")" + "|" + - r"(?P" + "edit|operational|versions" + r")" + + r"(?P" + "edit|operational|latest|build|versions" + r")" + r")" + r")" + r")" @@ -1348,7 +1348,16 @@ def is_mapping_spec(mapping): >>> is_mapping_spec("hst-cos-deadtab-edit") True - >>> is_mapping_spec("jwst-operational") + >>> is_mapping_spec("jwst-latest") + True + + >>> is_mapping_spec("latest") + True + + >>> is_mapping_spec("jwst-build") + True + + >>> is_mapping_spec("build") True >>> is_mapping_spec("hst-foo") @@ -1357,7 +1366,7 @@ def is_mapping_spec(mapping): >>> is_mapping_spec("hst_wfc3_0001.imap") True """ - return is_mapping(mapping) or (isinstance(mapping, str) and bool(CONTEXT_RE.match(mapping))) + return is_mapping(mapping) or (isinstance(mapping, str) and bool(CONTEXT_RE.match(mapping))) or mapping in ["latest", "build"] def is_context(mapping): """Return True IFF `mapping` has an extension indicating a CRDS CONTEXT, i.e. .pmap.""" diff --git a/crds/core/generic_tpn.py b/crds/core/generic_tpn.py index 09e611c2c..af52855d4 100644 --- a/crds/core/generic_tpn.py +++ b/crds/core/generic_tpn.py @@ -292,7 +292,10 @@ def load_all_type_constraints(observatory): to customize the more generalized constraints loaded later. """ from crds.core import rmap, heavy_client - pmap_name = heavy_client.load_server_info(observatory).operational_context + try: + pmap_name = heavy_client.load_server_info(observatory).latest_context + except AttributeError: + pmap_name = heavy_client.load_server_info(observatory).operational_context pmap = rmap.get_cached_mapping(pmap_name) locator = utils.get_locator_module(observatory) for instr in pmap.selections: diff --git a/crds/core/heavy_client.py b/crds/core/heavy_client.py index bc8c505b1..6beee8440 100644 --- a/crds/core/heavy_client.py +++ b/crds/core/heavy_client.py @@ -404,7 +404,7 @@ def get_context_name(observatory, context=None): """Return the .pmap name of the default context based on: 1. literal definitiion in `context` (jwst_0001.pmap) - 2. symbolic definition in `context` (jwst-operational or jwst-edit) + 2. symbolic definition in `context` (latest or jwst-edit) 3. date-based definition in `context` (jwst-2017-01-15T00:05:00) 4. CRDS_CONTEXT env var override @@ -415,23 +415,42 @@ def get_context_name(observatory, context=None): def get_final_context(info, context): """Based on env CRDS_CONTEXT, the `context` parameter, and the server's reported, - cached, or defaulted `operational_context`, choose the pipeline mapping which + cached, or defaulted `latest_context`, choose the pipeline mapping which defines the reference selection rules. Returns a .pmap name """ env_context = config.get_crds_env_context() - if context: # context parameter trumps all, -operational is default - input_context = context + try: + latest_context = str(info.latest_context) + except AttributeError: + latest_context = str(info.operational_context) + if context: # context parameter trumps all, -latest is default + if context == 'latest': + input_context = latest_context + elif context == 'build': + input_context = api.get_build_context(info.observatory) + else: + input_context = context log.verbose("Using reference file selection rules", srepr(input_context), "defined by caller.") info.status = "context parameter" elif env_context: - input_context = env_context + if env_context in ['latest', f"{info.observatory}-operational", f"{info.observatory}-latest"]: + input_context = latest_context + elif env_context in ['build', f"{info.observatory}-build"]: + input_context = api.get_build_context(info.observatory) + else: + input_context = env_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by environment CRDS_CONTEXT.") info.status = "env var CRDS_CONTEXT" else: - input_context = str(info.operational_context) + # Default for JWST if no env context and no explicit context is BUILD CONTEXT for installed jwst version + if info.observatory == 'jwst': + input_context = api.get_build_context('jwst') + # For other missions, default is latest (formerly operational) context + else: + input_context = latest_context log.verbose("Using reference file selection rules", srepr(input_context), "defined by", info.status + ".") final_context = translate_date_based_context(input_context, info.observatory) return final_context @@ -453,8 +472,10 @@ def translate_date_based_context(context, observatory=None): info = get_config_info(observatory) - if context == info.observatory + "-operational": - return info["operational_context"] + latest_context_names = [info.observatory + "-operational", info.observatory + "-latest", "latest"] + + if context in latest_context_names: + return info.get('latest_context', info['operational_context']) elif context == info.observatory + "-edit": return info["edit_context"] elif context == info.observatory + "-versions": diff --git a/crds/list.py b/crds/list.py index 5ac294981..0fb3c59b4 100644 --- a/crds/list.py +++ b/crds/list.py @@ -348,7 +348,10 @@ def add_args(self): self.add_argument("--tpns", nargs="*", dest="tpns", metavar="FILES", default=None, help="print the certify constraints (.tpn's) associated with the specified or implied files.") - + self.add_argument("--latest-context", action="store_true", dest="latest_context", + help="print the name of the latest context on the central CRDS server.") + self.add_argument("--build-context", action="store_true", dest="build_context", + help="print the name of the build context on the central CRDS server.") self.add_argument("--operational-context", action="store_true", dest="operational_context", help="print the name of the operational context on the central CRDS server.") self.add_argument("--remote-context", type=str, metavar="PIPELINE", @@ -375,12 +378,15 @@ def main(self): if self.args.file_properties is not None: # including [] return self.list_file_properties() - if self.args.operational_context: + if self.args.operational_context or self.args.latest_context: print(self.default_context) return if self.args.remote_context: print(self.remote_context) return + if self.args.build_context: + print(self.build_context) + return if self.args.resolve_contexts: self.list_resolved_contexts() @@ -410,7 +416,7 @@ def main(self): self.list_required_parkeys() return log.errors() - + def list_resolved_contexts(self): """Print out the literal interpretation of the contexts implied by the script's context specifiers. @@ -428,6 +434,12 @@ def remote_context(self): with log.error_on_exception("Failed resolving remote context"): return api.get_remote_context(self.observatory, self.args.remote_context) + @property + def build_context(self): + self.require_server_connection() + with log.error_on_exception("Failed resolving build context"): + return api.get_build_context(self.observatory) + @property def implied_files(self): """Return the filenames implied by a combination of the specified contexts @@ -656,9 +668,14 @@ def list_config(self): "version": heavy_client.version_info() }) _print_dict("CRDS Actual Paths", real_paths) - _print_dict("CRDS Server Info", server, - ["observatory", "status", "connected", "operational_context", "last_synced", - "reference_url", "mapping_url", "effective_mode"]) + try: + _print_dict("CRDS Server Info", server, + ["observatory", "status", "connected", "latest_context", "last_synced", + "reference_url", "mapping_url", "effective_mode"]) + except Exception: + _print_dict("CRDS Server Info", server, + ["observatory", "status", "connected", "operational_context", "last_synced", + "reference_url", "mapping_url", "effective_mode"]) download_uris = dict( config_uri = os.path.dirname(api.get_flex_uri("server_config")), pickle_uri = os.path.dirname(api.get_flex_uri("xyz.pmap.pkl")), diff --git a/crds/submit/submit.py b/crds/submit/submit.py index 26bdb9d9f..f317b22bc 100644 --- a/crds/submit/submit.py +++ b/crds/submit/submit.py @@ -106,7 +106,8 @@ def finish_parameters(self): locked_instrument=self.instrument, username=self.username, password=password, base_url=self.base_url) if self.args.derive_from_context is None: self.args.derive_from_context = self.observatory + "-edit" - if self.args.derive_from_context.endswith(("-edit", "-operational")): + self.args.derive_from_context.replace("-operational", "-latest") + if self.args.derive_from_context.endswith(("-edit", "-latest")): # this actually floats for concurrent deliveries self.pmap_mode = "pmap_" + self.args.derive_from_context.split("-")[-1] else: diff --git a/crds/sync.py b/crds/sync.py index 2be0d8706..c1d121ce7 100644 --- a/crds/sync.py +++ b/crds/sync.py @@ -401,7 +401,7 @@ def get_synced_references(self): return active_references def update_context(self): - """Update the CRDS operational context in the cache. Handle pipeline-specific + """Update the CRDS latest context in the cache. Handle pipeline-specific targeted features of (a) verifying a context switch as actually recorded in the local CRDS cache and (b) echoing/pushing the pipeline context back up to the CRDS server for tracking using an id/authorization key. @@ -410,7 +410,10 @@ def update_context(self): """ if not log.errors() or self.args.force_config_update: if self.args.verify_context_change: - old_context = heavy_client.load_server_info(self.observatory).operational_context + try: + old_context = heavy_client.load_server_info(self.observatory).latest_context + except AttributeError: + old_context = heavy_client.load_server_info(self.observatory).operational_context heavy_client.update_config_info(self.observatory) if self.args.verify_context_change: self.verify_context_change(old_context) @@ -444,7 +447,10 @@ def server_info(self): def verify_context_change(self, old_context): """Verify that the starting and post-sync contexts are different, or issue an error.""" - new_context = heavy_client.load_server_info(self.observatory).operational_context + try: + new_context = heavy_client.load_server_info(self.observatory).latest_context + except AttributeError: + new_context = heavy_client.load_server_info(self.observatory).operational_context if old_context == new_context: log.error("Expected operational context switch but starting and post-sync contexts are both", repr(old_context)) else: @@ -452,12 +458,18 @@ def verify_context_change(self, old_context): def push_context(self): """Push the final context recorded in the local cache to the CRDS server so it can be displayed - as the operational state of a pipeline. + as the latest state of a pipeline. """ info = heavy_client.load_server_info(self.observatory) - with log.error_on_exception("Failed pushing cached operational context name to CRDS server"): - api.push_remote_context(self.observatory, "operational", self.args.push_context, info.operational_context) - log.info("Pushed cached operational context name", repr(info.operational_context), "to CRDS server") + try: + ctx = 'latest' + latest_context = info.latest_context + except AttributeError: + ctx = 'operational' + latest_context = info.operational_context + with log.error_on_exception(f"Failed pushing cached {ctx} context name to CRDS server"): + api.push_remote_context(self.observatory, ctx, self.args.push_context, latest_context) + log.info(f"Pushed cached {ctx} context name", repr(latest_context), "to CRDS server") # ------------------------------------------------------------------------------------------