From 12fcd65ba7945c89e663cddf5fc79bd628da1521 Mon Sep 17 00:00:00 2001 From: Matthieu Leclair Date: Tue, 12 Dec 2023 14:27:30 +0100 Subject: [PATCH 001/245] Make jobs fit for async (#52) * add: Make icon.py fit for async * ref: don't use arguments to main, rather store info in config * GitHub Action: Apply Pep8-formatting * add:ref: model becomes workflow + get job dependencies from cfg - No need to travel with `model_cfg`. The relevant part is now in `cfg.worflow` and the former `cfg.model` is now `cfg.workflow_name` - A method is added to Config to generate the dependency part of the sbatch command - Some other minor improvements like recursive abs path or recursive attributes from dictionnaries of any depth - NOTHING TESTED YET !!! * fix: leftover from model => workflow_name * ref: remove dependencies for prepare_data, not needed * fix:add: get_dep_cmd was buggy, add wait_for_previous method * GitHub Action: Apply Pep8-formatting * add: cycling mechanism for async workflows * GitHub Action: Apply Pep8-formatting * ref: some reorganization of files - move `class Config` in its own `config.py` - mov workflow.yaml one level up becuase the `config` dir becomes confusing with `config.py`. Also not really needed to have that folder with only this file in it. * GitHub Action: Apply Pep8-formatting * ref: start using pathlib and make 2 introspective methods recurive * ref: continue using pathlib, remove work_root arg (already in cfg) * GitHub Action: Apply Pep8-formatting * ref(prepare_data): continue using pathlib, replace setattr * ref:add:fix: some more slurm integration - use `is_async` instead of python keyword `async` - add potential user provided dependency to `get_dep_xxx` methods. This allows for internal dependencies between subjobs of a particular task, like prepare_data. - add a submit method to the `Config` class - Start slurm integration for prepare_data. Continue the same way for the rest of it. - replace some `setattr` calls by direct assignments - move `jobs`, `startdate` and `enddate` to the Config object and remove them from the arguments of `restart_runs_xxx` functions --------- Co-authored-by: github-actions --- cases/cosmo-ghg-spinup-test/config.yaml | 2 +- cases/cosmo-ghg-test/config.yaml | 2 +- cases/icon-art-global-test/config.yaml | 2 +- cases/icon-art-oem-test/config.yaml | 2 +- cases/icon-test/config.yaml | 2 +- config.py | 410 +++++++++++++++ config/models.yaml | 24 - jobs/biofluxes.py | 4 +- jobs/check_output.py | 2 +- jobs/cosmo.py | 20 +- jobs/emissions.py | 6 +- jobs/icon.py | 50 +- jobs/int2lm.py | 10 +- jobs/obs_nudging.py | 2 +- jobs/octe.py | 2 +- jobs/oem.py | 6 +- jobs/online_vprm.py | 2 +- jobs/photo_rate.py | 2 +- jobs/post_cosmo.py | 6 +- jobs/post_int2lm.py | 4 +- jobs/prepare_data.py | 111 ++-- jobs/reduce_output.py | 2 +- jobs/tools/check_model.py | 6 +- jobs/verify_chain.py | 2 +- run_chain.py | 653 +++++------------------- workflows.yaml | 27 + 26 files changed, 703 insertions(+), 658 deletions(-) create mode 100644 config.py delete mode 100644 config/models.yaml create mode 100644 workflows.yaml diff --git a/cases/cosmo-ghg-spinup-test/config.yaml b/cases/cosmo-ghg-spinup-test/config.yaml index 2351b766..18749b8a 100644 --- a/cases/cosmo-ghg-spinup-test/config.yaml +++ b/cases/cosmo-ghg-spinup-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'cosmo-ghg-spinup-test' case with COSMO-GHG -model: cosmo-ghg +workflow: cosmo-ghg constraint: gpu ntasks_per_node: 12 restart_step: PT6H diff --git a/cases/cosmo-ghg-test/config.yaml b/cases/cosmo-ghg-test/config.yaml index dc1134e9..a579f1de 100644 --- a/cases/cosmo-ghg-test/config.yaml +++ b/cases/cosmo-ghg-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'cosmo-ghg-test' case with COSMO-GHG -model: cosmo-ghg +workflow: cosmo-ghg constraint: gpu ntasks_per_node: 12 restart_step: PT6H diff --git a/cases/icon-art-global-test/config.yaml b/cases/icon-art-global-test/config.yaml index 2344a8df..9a39aaef 100644 --- a/cases/icon-art-global-test/config.yaml +++ b/cases/icon-art-global-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'icon-art-global-test' case with ICON -model: icon-art-global +workflow: icon-art-global constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index 5a3d8511..3be7d53d 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'icon-art-oem-test' case with ICON -model: icon-art-oem +worflow: icon-art-oem constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index e1d58782..8b9f497f 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'icon-test' case with ICON -model: icon +workflow: icon constraint: gpu run_on: cpu compute_queue: normal diff --git a/config.py b/config.py new file mode 100644 index 00000000..299f7220 --- /dev/null +++ b/config.py @@ -0,0 +1,410 @@ +import subprocess +import os +from pathlib import Path + + +class Config(): + + def __init__(self, casename): + """Initialize an instance of the Config class. + + Initializes an instance of the Config class with user-specific + and default attributes. The class represents a processing chain for a + particular case, and its attributes are populated based on the provided + `casename`. + + Parameters + ---------- + casename : str + The identifier for the case, typically specifying the configuration + and settings to be used in the processing chain. + + Attributes + ---------- + user_name : str + The username of the current user, obtained from the 'USER' environment variable. + email : str + The user's email address, initially set to None and updated using the `set_email` method. + casename : str + The specified case name for the processing chain. + chain_src_dir : str + The source directory for the processing chain, typically the current working directory. + case_path : str + The path to the case directory under 'cases/' for the specified `casename`. + work_root : str + The root directory for processing chain execution, typically located under the source directory. + + Notes + ----- + The method also loads user-defined attributes from the configuration file, + sets specific settings based on the node type ('gpu' or 'mc'), and initializes + other instance-specific attributes. + """ + # Global attributes (initialized with default values) + self.user_name = os.environ['USER'] + self.set_email() + self.casename = casename + self.set_account() + + self.chain_src_dir = Path.cwd() + self.case_path = self.chain_src_dir / 'cases' / casename + self.work_root = self.chain_src_dir / 'work' + + # User-defined attributes from config file + self.load_config_file(casename) + + # Specific settings based on the node type ('gpu' or 'mc') + self.set_node_info() + + # Set workflow and async attributes and initiate job ids dict + self.set_workflow() + + def load_config_file(self, casename): + """Load configuration settings from a YAML file and set them as attributes. + + This method reads the configuration settings from a YAML file located in + the 'cases/casename' directory and sets them as attributes of the instance. + + Parameters + ---------- + casename : str + Name of the folder in 'cases/' where the configuration files are stored. + + Returns + ------- + Config + The same `Config` instance with configuration settings as attributes. + + Raises + ------ + FileNotFoundError + If the specified configuration file or case directory is not found. + + Notes + ----- + If the configuration file does not exist, the method will attempt to suggest + a similar case directory based on a Levenshtein distance comparison with + existing case directories. The method directly assigns values from the + configuration file to instance attributes for easy access. + """ + cfg_file = Path('cases', casename, 'config.yaml').resolve() + + if not cfg_file.is_file(): + all_cases = [ + path.name for path in os.scandir('cases') if path.is_dir() + ] + closest_name = min([(tools.levenshtein(casename, name), name) + for name in all_cases], + key=lambda x: x[0])[1] + raise FileNotFoundError( + f"Case-directory '{casename}' not found, did you mean '{closest_name}'?" + ) + + try: + with cfg_file.open('r') as yaml_file: + cfg_data = yaml.load(yaml_file, Loader=yaml.FullLoader) + except FileNotFoundError: + raise FileNotFoundError( + f"No file 'config.yaml' in {cfg_file.parent}") + + # Directly assign values to instance attributes + for key, value in cfg_data.items(): + setattr(self, key, value) + + # rename the workflow attribute to avoid name clash + self.workflow_name = self.workflow + + return self + + def set_account(self): + """Set the compute account based on user information. + + This method determines the compute account to be used based on the user's + name and system configuration. + + Returns + ------- + Config + The same `Config` instance with the `compute_account` attribute set. + + Notes + ----- + - If the user name is 'jenkins', the compute account is set to 'g110' for + Jenkins testing. + - If an account is specified in the user's '~/.acct' file, it will be used + as the compute account. + - If neither of the above conditions is met, the standard account is + determined using the 'id -gn' command. + """ + if self.user_name == 'jenkins': + # g110 account for Jenkins testing + self.compute_account = 'g110' + elif (p := Path.home() / '.acct').exists(): + # Use account specified in ~/.acct file + with p.open('r') as file: + self.compute_account = file.read().rstrip() + else: + # Use standard account + self.compute_account = os.popen("id -gn").read().splitlines()[0] + + def set_node_info(self): + """Set node-specific information based on configuration settings. + + This method configures node-specific settings, such as the number of tasks + per node and CUDA-related environment variables, based on the provided + configuration settings in the instance. + + Returns + ------- + Config + The same `Config` instance with updated node-specific attributes. + + Raises + ------ + ValueError + If the 'constraint' or 'run_on' configuration values are invalid. + """ + if self.constraint == 'gpu': + if self.workflow_name.startswith('icon'): + if self.run_on == 'gpu': + self.ntasks_per_node = 1 + elif self.run_on == 'cpu': + self.ntasks_per_node = 12 + else: + raise ValueError( + "Invalid value for 'run_on' in the configuration." + "It should be either 'gpu' or 'cpu'.") + else: + self.ntasks_per_node = 12 + self.mpich_cuda = ('export MPICH_RDMA_ENABLED_CUDA=1\n' + 'export MPICH_G2G_PIPELINE=256\n' + 'export CRAY_CUDA_MPS=1\n') + elif self.constraint == 'mc': + self.ntasks_per_node = 36 + self.mpich_cuda = '' + else: + raise ValueError( + "Invalid value for 'constraint' in the configuration." + "It should be either 'gpu' or 'mc'.") + + def set_workflow(self): + """set workflow and async attr, initiate job ids dict""" + + with open('workflows.yaml') as file: + workflows = yaml.safe_load(file) + self.workflow = workflows[self.workflow_name] + self.is_async = 'dependencies' in self.workflow + + # Initiate empty job ids dictionnary so that it can be filled in later + self.job_ids = {'current': {}, 'previous': {}} + + def set_restart_step_hours(self): + """Set the restart step in hours. + + Converts the 'restart_step' attribute, which is in ISO8601 duration format, + to hours and stores the result in the 'restart_step_hours' attribute. + + Returns + ------- + Config + The same `Config` instance with the 'restart_step_hours' attribute set. + """ + self.restart_step_hours = int( + tools.iso8601_duration_to_hours(self.restart_step)) + + def set_email(self): + """Set the user's email address based on system configuration. + + This method determines the user's email address based on the user's name + and system configuration. + + Returns + ------- + Config + The same `Config` instance with the `user_mail` attribute set. + + Notes + ----- + - If the user name is 'jenkins', the user's email address is set to None. + - If an email address is specified in the user's '~/.forward' file, it will + be used as the user's email address. + - If neither of the above conditions is met, the user's email address is set + to None. + """ + if self.user_name == 'jenkins': + self.user_mail = None + elif (p := Path.home() / '.forward').exists(): + with p.open('r') as file: + self.user_mail = file.read().rstrip() + else: + self.user_mail = None + + def print_config(self): + """Print the configuration attributes and their values. + + This method displays the configuration attributes and their corresponding + values in a formatted manner. Lists and dictionaries within the configuration + are also displayed with appropriate indentation. + + Notes + ----- + - The maximum column width for the attribute names is automatically determined. + - The method prints the attribute name, its type, and its value. + - If an attribute is a list, it is displayed with each item indented. + - If an attribute is a dictionary, it is also displayed with each key-value + pair indented. + """ + # max_col_width = max(len(key) for key in vars(self)) + 1 + max_col_width = 27 + + print("\nConfiguration:") + print(f"{'Attribute':<{max_col_width}} Type Value") + print("-" * 80) + for key, value in vars(self).items(): + if isinstance(value, list): + # If the value is a list, format it with indentation + print(f"{key:<{max_col_width}} list") + for item in value: + item_type = type(item).__name__ + print(f" - {item:<{max_col_width-4}} {item_type}") + elif isinstance(value, dict): + # If the value is a dictionary, format it as before + print(f"{key:<{max_col_width}} dict") + for sub_key, sub_value in value.items(): + sub_value_type = type(sub_value).__name__ + print( + f" - {sub_key:<{max_col_width-4}} {sub_value_type:<4} {sub_value}" + ) + else: + # Standard output + key_type = type(key).__name__ + print(f"{key:<{max_col_width}} {key_type:<4} {value}") + + def convert_paths_to_absolute(self, dct=None): + """Convert relative file paths to absolute paths in the configuration. + + Recursively convert all strings starting with './' in the instance + attributes to absolute paths. + + Returns + ------- + Config + The same `Config` instance with relative file paths converted to absolute paths. + """ + if dct is None: + self.convert_paths_to_absolute(dct=vars(self)) + else: + for k, v in dct.items(): + if isinstance(v, dict): + self.convert_paths_to_absolute(dct=v) + elif isinstance(v, str) and v.startswith('./'): + dct[k] = Path(v).absolute() + + def create_vars_from_dicts(self, dct=None, key=None): + """Create instance attributes from dictionary entries in the configuration. + + This method recursively iterates through the instance's attribute dictionary + and checks for dictionary values. For each dictionary encountered, it creates + new instance attributes by concatenating the original attribute name and the + dictionary key, and assigns the corresponding values. + + Returns + ------- + Config + The same `Config` instance with new attributes created from dictionary entries. + """ + + if dct is None: + self.create_vars_from_dicts(dct=vars(self).copy()) + else: + for k, v in dct.items(): + subkey = k if key is None else key + '_' + k + if isinstance(v, dict): + self.create_vars_from_dicts(dct=v, key=subkey) + else: + setattr(self, subkey, v) + + def get_dep_ids(self, job_name, add_dep=None): + """Get dependency job ids for `job_name`""" + + # Initial list of dependencies + if add_dep is not None: + if type(add_dep) is int: + dep_id_list = [add_dep] + else: + try: + dep_id_list = list(add_dep) + except TypeError: + print(f'add_dep must be an iterable') + else: + dep_id_list = [] + + # Add job dependencies + if self.is_async: + # Could be that job has no dependency, even in an async config, + # e.g., prepare_data + if deps := self.workflow['dependencies'].get(job_name): + for stage in 'previous', 'current': + if dep_stage := deps.get(stage): + for job in dep_stage: + # Could be that dep job id does not exist, e.g., + # if dep job is deactivated or it's the first chunk + if dep_id := self.job_ids[stage].get(job): + dep_id_list.extend(dep_id) + return dep_id_list + + def get_dep_cmd(self, job_name, add_dep=None): + """Generate the part of the sbatch command that sepcifies dependencies for job_name.""" + + if self.is_async: + # async case + if dep_ids := self.get_dep_ids(job_name, add_dep=add_dep): + dep_str = ':'.join(map(str, deps_ids)) + return f'--dependency=afterok:{dep_str}' + else: + # job_name has no dependencies but still belongs to an async workflow + # so don't use --wait + return None + else: + # sequential case + return '--wait' + + def submit(self, job_name, script, add_dep=None): + """Submit job with dependencies""" + + script_path = Path(script) + sbatch_cmd = ['sbatch', '--parsable'] + if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): + sbatch_cmd.append(dep_cmd) + sbatch_cmd.append(script_path.name) + + result = subprocess.run(sbatch_cmd, cwd=script_path.parent, capture_output=True) + job_id = int(result.stdout) + if not job_name in self.job_ids['current']: + self.job_ids['current'][job_name] = [job_id] + else: + self.job_ids['current'][job_name].append(job_id) + + # If needed internaly in a multi-job task like prepare_data + # Can then be passed as add_dep keyword + return job_id + + def wait_for_previous(self): + """wait for all jobs of the previous stage to be finished + + Do this by submitting a fake job depending on all jobs from the 'previous' stage. + """ + + dep_ids = [] + for ids in self.job_ids['previous'].values(): + dep_ids.extend(ids) + if dep_ids: + job_file = 'submit.wait.slurm' + with open(job_file, mode='w') as wait_job: + wait_job.write("""#!/bin/bash\n#Do nothing\nexit 0""") + + subprocess.run([ + 'sbatch', '-W', '--nodes=1', '--job-name=wait', + f'--account={self.compute_account}', job_file + ], + check=True) diff --git a/config/models.yaml b/config/models.yaml deleted file mode 100644 index a19f188c..00000000 --- a/config/models.yaml +++ /dev/null @@ -1,24 +0,0 @@ -models: - cosmo: - jobs: ['prepare_data', 'int2lm', 'cosmo', 'post_cosmo'] - features: ['restart', 'spinup'] - cosmo-ghg: - jobs: ['prepare_data', 'emissions', 'biofluxes', 'oem', 'online_vprm', - 'int2lm', 'post_int2lm', 'cosmo', 'post_cosmo'] - features: ['restart', 'tracers', 'spinup'] - cosmo-art: - jobs: ['prepare_data', 'emissions', 'obs_nudging', 'photo_rate', - 'int2lm', 'cosmo', 'post_cosmo'] - features: ['nesting', 'spinup'] - icon: - jobs: ['prepare_data', 'icon'] - features: ['restart'] - icon-art: - jobs: ['prepare_data', 'icon'] - features: ['restart'] - icon-art-global: - jobs: ['prepare_data', 'icon'] - features: ['restart'] - icon-art-oem: - jobs: ['prepare_data', 'icon'] - features: ['restart'] diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 78471f09..ffb39a5c 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -19,7 +19,7 @@ from . import tools, prepare_data -def main(cfg, model_cfg): +def main(cfg): """Prepare the biofluxes-files for the simulation. Only necessary for **COSMO** simulations. @@ -33,7 +33,7 @@ def main(cfg, model_cfg): Object holding all user-configuration parameters as attributes """ tools.check_model(cfg, 'cosmo-ghg') - cfg = prepare_data.set_cfg_variables(cfg, model_cfg) + cfg = prepare_data.set_cfg_variables(cfg) scratch_path = os.path.join(cfg.int2lm_input, 'vprm') diff --git a/jobs/check_output.py b/jobs/check_output.py index 133e969c..a6724ccb 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -699,7 +699,7 @@ def create_animations(cfg): duration=300) -def main(cfg, model_cfg): +def main(cfg): """Checks output variables whether they are in a phyiscally reasonable range. diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 4c43e815..5897adef 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -20,7 +20,7 @@ from datetime import datetime, timedelta -def set_cfg_variables(cfg, model_cfg): +def set_cfg_variables(cfg): setattr(cfg, 'cosmo_base', os.path.join(cfg.chain_root, 'cosmo')) setattr(cfg, 'cosmo_input', os.path.join(cfg.chain_root, 'cosmo', 'input')) setattr(cfg, 'cosmo_run', os.path.join(cfg.chain_root, 'cosmo', 'run')) @@ -30,7 +30,7 @@ def set_cfg_variables(cfg, model_cfg): os.path.join(cfg.chain_root, 'cosmo', 'output_reduced')) # Number of tracers - if 'tracers' in model_cfg['models'][cfg.model]['features']: + if 'tracers' in cfg.workflow['features']: tracer_csvfile = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, 'cosmo_tracers.csv') if os.path.isfile(tracer_csvfile): @@ -62,7 +62,7 @@ def set_cfg_variables(cfg, model_cfg): return cfg -def main(cfg, model_cfg): +def main(cfg): """Setup the namelists for a **COSMO** tracer run and submit the job to the queue @@ -93,7 +93,7 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = set_cfg_variables(cfg, model_cfg) + cfg = set_cfg_variables(cfg) logfile = os.path.join(cfg.log_working_dir, "cosmo") logfile_finish = os.path.join(cfg.log_finished_dir, "cosmo") @@ -176,23 +176,23 @@ def main(cfg, model_cfg): # Create restart directory if feature is present and # if there is no spinup - if 'restart' in model_cfg['models'][cfg.model]['features'] and not \ + if 'restart' in cfg.workflow['features'] and not \ hasattr(cfg, 'spinup'): tools.create_dir(cfg.cosmo_restart_out, "cosmo_restart_out") # Copy cosmo executable - cfg.cosmo['execname'] = cfg.model.lower() + cfg.cosmo['execname'] = cfg.workflow_name.lower() tools.copy_file(cfg.cosmo['binary_file'], os.path.join(cfg.cosmo_run, cfg.cosmo['execname'])) # Prepare namelist and submit job tracer_csvfile = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, 'cosmo_tracers.csv') - if cfg.model == 'cosmo': + if cfg.workflow_name == 'cosmo': namelist_names = ['ORG', 'IO', 'DYN', 'PHY', 'DIA', 'ASS', 'SAT'] - elif cfg.model == 'cosmo-ghg': + elif cfg.workflow_name == 'cosmo-ghg': namelist_names = ['AF', 'ORG', 'IO', 'DYN', 'GHG', 'PHY', 'DIA', 'ASS'] - elif cfg.model == 'cosmo-art': + elif cfg.workflow_name == 'cosmo-art': namelist_names = [ 'ART', 'ASS', 'DIA', 'DYN', 'EPS', 'INI', 'IO', 'ORG', 'PHY' ] @@ -231,7 +231,7 @@ def main(cfg, model_cfg): # Append INPUT_GHG namelist with tracer definitions from csv file if os.path.isfile(tracer_csvfile): - if cfg.model == 'cosmo-ghg': + if cfg.workflow_name == 'cosmo-ghg': input_ghg_filename = os.path.join(cfg.cosmo_run, 'INPUT_GHG') write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg) diff --git a/jobs/emissions.py b/jobs/emissions.py index cf98c92f..5f3a9b70 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -20,7 +20,7 @@ from . import tools, prepare_data -def main(cfg, model_cfg): +def main(cfg): """Copy emission files to the **int2lm** input directory. Necessary for both **COSMO** and **COSMOART** simulations. @@ -49,7 +49,7 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = prepare_data.set_cfg_variables(cfg, model_cfg) + cfg = prepare_data.set_cfg_variables(cfg) dest_prefix = "emis_" if not isinstance(cfg.emissions['dir'], list): @@ -84,5 +84,5 @@ def main(cfg, model_cfg): # convert grid_mapping_name from string (NF90_STRING) to char # (NF90_CHAR) (needed for int2lm to work) - if cfg.model.startswith('cosmo'): + if cfg.workflow_name.startswith('cosmo'): tools.string2char.main(dest_path) diff --git a/jobs/icon.py b/jobs/icon.py index a63c835b..4e6b8dcb 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -17,7 +17,7 @@ from . import tools, prepare_data -def main(cfg, model_cfg): +def main(cfg): """Setup the namelists for an **ICON** tracer run and submit the job to the queue @@ -49,7 +49,7 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = prepare_data.set_cfg_variables(cfg, model_cfg) + cfg = prepare_data.set_cfg_variables(cfg) logfile = os.path.join(cfg.log_working_dir, "icon") logfile_finish = os.path.join(cfg.log_finished_dir, "icon") @@ -77,27 +77,25 @@ def main(cfg, model_cfg): cfg.meteo['nameformat']) + '.nc') # Write run script (run_icon.job) - icon_runjob = os.path.join(cfg.case_path, cfg.icon_runjob_filename) - with open(icon_runjob) as input_file: - to_write = input_file.read() - output_file = os.path.join(cfg.icon_work, "run_icon.job") - with open(output_file, "w") as outf: - outf.write( - to_write.format(cfg=cfg, - inidata_filename=inidata_filename, - logfile=logfile, - logfile_finish=logfile_finish)) - - result = subprocess.run( - ["sbatch", "--wait", - os.path.join(cfg.icon_work, 'run_icon.job')]) - exitcode = result.returncode - - # In case of ICON-ART, ignore the "invalid pointer" error on successful run - if cfg.model.startswith('icon-art'): - if tools.grep("free(): invalid pointer", logfile)['success'] and \ - tools.grep("clean-up finished", logfile)['success']: - exitcode = 0 - - if exitcode != 0: - raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + template = (cfg.case_path / cfg.icon_runjob_filename).read_text() + script_str = template.format(cfg=cfg, + inidata_filename=inidata_filename, + logfile=logfile, + logfile_finish=logfile_finish) + script = (cfg.icon_work / 'run_icon.job').write_text(script_str) + + # Submit run script + cfg.submit('icon', script) + + # Anything hapenning after submission only makes sense in sequential mode + if not cfg.is_async: + exitcode = result.returncode + + # In case of ICON-ART, ignore the "invalid pointer" error on successful run + if cfg.workflow_name.startswith('icon-art'): + if tools.grep("free(): invalid pointer", logfile)['success'] and \ + tools.grep("clean-up finished", logfile)['success']: + exitcode = 0 + + if exitcode != 0: + raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index da94a56c..59099c52 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -18,7 +18,7 @@ from . import tools, prepare_data -def set_cfg_variables(cfg, model_cfg): +def set_cfg_variables(cfg): setattr(cfg, 'int2lm_run', os.path.join(cfg.chain_root, 'int2lm', 'run')) setattr(cfg, 'int2lm_output', @@ -27,7 +27,7 @@ def set_cfg_variables(cfg, model_cfg): return cfg -def main(cfg, model_cfg): +def main(cfg): """Setup the namelist for **int2lm** and submit the job to the queue. Necessary for both **COSMO** and **COSMOART** simulations. @@ -64,8 +64,8 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = prepare_data.set_cfg_variables(cfg, model_cfg) - cfg = set_cfg_variables(cfg, model_cfg) + cfg = prepare_data.set_cfg_variables(cfg) + cfg = set_cfg_variables(cfg) # Total number of processes np_tot = cfg.int2lm['np_x'] * cfg.int2lm['np_y'] @@ -89,7 +89,7 @@ def main(cfg, model_cfg): extpar_dir) # Copy landuse and plant-functional-type files - if cfg.model == 'cosmo-art': + if cfg.workflow_name == 'cosmo-art': lu_file_src = cfg.int2lm['lu_file'] lu_file_dst = os.path.join(extpar_dir, 'landuse.nc') tools.copy_file(lu_file_src, lu_file_dst) diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index bf2557cb..fcb9a0f8 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -9,7 +9,7 @@ from . import tools -def main(cfg, model_cfg): +def main(cfg): """Copy and rename the obs_nudging files to the **COSMO** input directory. In the folder ``cfg.obs_nudging_dir``, the files are saved in the format diff --git a/jobs/octe.py b/jobs/octe.py index b52f2a03..6c6acf22 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -161,7 +161,7 @@ def perturb_bgs_in_dir(lambdas_nc, directory): entry.name)) -def main(cfg, model_cfg): +def main(cfg): """Copy necessary input files for **COSMO** and perturb BG. Copies the NetCDF-files found at cfg.octe_maps and cfg.octe_lambdas to diff --git a/jobs/oem.py b/jobs/oem.py index 358af4e9..0aad7534 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -8,7 +8,7 @@ from . import tools, cosmo -def main(cfg, model_cfg): +def main(cfg): """Copy emission and profile files to the **cosmo** or **icon** input directory. @@ -23,7 +23,7 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = cosmo.set_cfg_variables(cfg, model_cfg) + cfg = cosmo.set_cfg_variables(cfg) oem_dir = cfg.oem['dir'] oem_gridded_emissions_nc = os.path.join(oem_dir, @@ -51,7 +51,7 @@ def main(cfg, model_cfg): raise RuntimeError("At least one of (hod/dow/moy) or (hoy) netcdfs " " have to be given for online emissions") - if cfg.model.startswith('icon'): + if cfg.workflow_name.startswith('icon'): input_dir = cfg.icon_input else: input_dir = cfg.cosmo_input diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 106457cf..340b14e8 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -8,7 +8,7 @@ from . import tools -def main(cfg, model_cfg): +def main(cfg): """Copy MODIS surface reflectance data and vegatation class fraction file to the **cosmo** input directory. diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index 7f8360b2..b7724d38 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -7,7 +7,7 @@ from . import tools -def main(cfg, model_cfg): +def main(cfg): """Copy photolysis-rate file to the **COSMOART** input directory. Only necessary for **COSMOART** simulations. diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 56d4dc1d..07025fe2 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -49,7 +49,7 @@ def runscript_commands_template(): ]) -def main(cfg, model_cfg): +def main(cfg): """Copy the output of a **COSMO**-run to a user-defined position. Write a runscript to copy all files (**COSMO** settings & output, @@ -67,8 +67,8 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = int2lm.set_cfg_variables(cfg, model_cfg) - cfg = cosmo.set_cfg_variables(cfg, model_cfg) + cfg = int2lm.set_cfg_variables(cfg) + cfg = cosmo.set_cfg_variables(cfg) logfile = os.path.join(cfg.log_working_dir, "post_cosmo") cosmo_run_dir = cfg.cosmo_run diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 787f7907..07b24f5b 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -9,7 +9,7 @@ from . import tools, int2lm -def main(cfg, model_cfg): +def main(cfg): """Combine multiple **int2lm** tracer-output files into a single one for **COSMO**. @@ -28,7 +28,7 @@ def main(cfg, model_cfg): cfg : config-object Object holding all user-configuration parameters as attributes """ - cfg = int2lm.set_cfg_variables(cfg, model_cfg) + cfg = int2lm.set_cfg_variables(cfg) # Int2lm processing always starts at hstart=0, thus modifying inidate inidate_int2lm_yyyymmddhh = cfg.startdate_sim_yyyymmddhh diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 894d2563..7f9e1533 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -25,6 +25,7 @@ # 2021-11-12 Modified for ICON-ART-simulations (mjaehn) import os +from pathlib import Path import logging import shutil import subprocess @@ -37,43 +38,36 @@ from calendar import monthrange -def set_cfg_variables(cfg, model_cfg): - # TODO: Change setattr() to direct assignment - if cfg.model.startswith('cosmo'): - setattr(cfg, 'int2lm_root', os.path.join(cfg.chain_root, 'int2lm')) - setattr(cfg, 'int2lm_input', os.path.join(cfg.int2lm_root, 'input')) - elif cfg.model.startswith('icon'): - setattr(cfg, 'icon_base', os.path.join(cfg.chain_root, 'icon')) - setattr(cfg, 'icon_input', os.path.join(cfg.chain_root, 'icon', - 'input')) - setattr(cfg, 'icon_input_icbc', - os.path.join(cfg.chain_root, 'icon', 'input', 'icbc')) - setattr(cfg, 'icon_work', os.path.join(cfg.chain_root, 'icon', 'run')) - setattr(cfg, 'icon_output', - os.path.join(cfg.chain_root, 'icon', 'output')) - setattr(cfg, 'icon_output_reduced', - os.path.join(cfg.chain_root, 'icon', 'output_reduced')) - setattr(cfg, 'icon_restart_out', - os.path.join(cfg.chain_root, 'icon', 'restart')) - setattr(cfg, 'icon_restart_in', - os.path.join(cfg.chain_root_prev, 'icon', 'run')) - setattr(cfg, 'icon_input_icbc_prev', - os.path.join(cfg.chain_root_prev, 'icon', 'input', 'icbc')) +def set_cfg_variables(cfg): + if cfg.workflow_name.startswith('cosmo'): + cfg.int2lm_root = cfg.chain_root / 'int2lm' + cfg.int2lm_input = cfg.int2lm_root / 'input' + elif cfg.workflow_name.startswith('icon'): + cfg.icon_base = cfg.chain_root / 'icon' + cfg.icon_input = cfg.icon_base / 'input' + cfg.icon_input_icbc = cfg.icon_input / 'icbc' + cfg.icon_work = cfg.icon_base / 'run' + cfg.icon_output = cfg.icon_base / 'output' + cfg.icon_output_reduced = cfg.icon_base / 'output_reduced' + cfg.icon_restart_out = cfg.icon_base / 'restart' + cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' + cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' cfg.input_files_scratch = {} - for varname in cfg.input_files: - cfg.input_files_scratch[varname] = os.path.join( - cfg.icon_input, os.path.basename(cfg.input_files[varname])) + for dsc, file in cfg.input_files.items(): + cfg.input_files[dsc] = (p := Path(file)) + cfg.input_files_scratch[dsc] = cfg.icon_input / p.name + cfg.create_vars_from_dicts() cfg.ini_datetime_string = cfg.startdate.strftime('%Y-%m-%dT%H:00:00Z') cfg.end_datetime_string = cfg.enddate.strftime('%Y-%m-%dT%H:00:00Z') - if cfg.model == 'icon-art-oem': + if cfg.workflow_name == 'icon-art-oem': cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime( '%Y%m%d_%H') - if cfg.model == 'icon-art-global': + if cfg.workflow_name == 'icon-art-global': # Nudge type (global or nothing) cfg.nudge_type = 2 if cfg.era5_global_nudging else 0 # Time step for global nudging in seconds @@ -83,15 +77,18 @@ def set_cfg_variables(cfg, model_cfg): if cfg.lrestart == '.TRUE.': cfg.restart_filename = 'restart_atm_DOM01.nc' - cfg.restart_file = os.path.join(cfg.icon_restart_in, - cfg.restart_filename) - cfg.restart_file_scratch = os.path.join(cfg.icon_work, - cfg.restart_filename) + cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename + cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename + + cfg.job_ids['current']['prepare_data'] = [] - return cfg +def async_error(cfg, part="This part"): + if cfg.is_async: + raise NotImplementedError(f"{part} isn't ready for async execution yet") -def main(cfg, model_cfg): + +def main(cfg): """ **ICON** @@ -131,9 +128,9 @@ def main(cfg, model_cfg): Object holding all user-configuration parameters as attributes """ - cfg = set_cfg_variables(cfg, model_cfg) + set_cfg_variables(cfg) - if cfg.model.startswith('icon'): + if cfg.workflow_name.startswith('icon'): logging.info('ICON input data (IC/BC)') #----------------------------------------------------- @@ -147,13 +144,29 @@ def main(cfg, model_cfg): #----------------------------------------------------- # Copy input files #----------------------------------------------------- - for varname in cfg.input_files: - varname_scratch = f'{varname}_scratch' - tools.copy_file(cfg.input_files[varname], - cfg.input_files_scratch[varname], - output_log=True) - - if cfg.model == 'icon-art-global': + wall_time = getattr(cfg, 'copy_input_walltime', '00:01:00') + queue = getattr(cfg, 'copy_input_queue', 'normal') + + script_lines = ['#!/usr/bin/env bash', + f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', + f'#SBATCH --account={cfg.compute_account}', + f'#SBATCH --time={walltime}', + f'#SBATCH --partition={queue}', + '#SBATCH --constraint=gpu', + '#SBATCH --nodes=1', + ''] + for target, destination in zip(cfg.input_files.values(), + cfg.input_files_scratch.values()): + script_lines.append(f'rsync -av {target} {destination}') + + + with (script := cfg.icon_base / 'copy_input.job').open('w') as f: + f.write('\n'.join(script_lines)) + + cfg.submit('prepare_data', script) + + if cfg.workflow_name == 'icon-art-global': + async_error(cfg, part='global ICON-ART') # -- Download ERA5 data and create the inicond file if cfg.era5_inicond and cfg.lrestart == '.FALSE.': # -- Fetch ERA5 data @@ -314,6 +327,7 @@ def main(cfg, model_cfg): process.communicate() else: # non-global ICON-ART + async_error(cfg, part='non-global ICON-ART') #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) #----------------------------------------------------- @@ -325,7 +339,7 @@ def main(cfg, model_cfg): meteo_file = os.path.join( cfg.icon_input_icbc, cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) - if cfg.model == 'icon-art' or cfg.model == 'icon-art-oem': + if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': chem_file = os.path.join( cfg.icon_input_icbc, cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) @@ -418,7 +432,7 @@ def main(cfg, model_cfg): #----------------------------------------------------- # Add Q (copy of QV) and/or PS to initial file #----------------------------------------------------- - if cfg.model.startswith('icon-art'): + if cfg.workflow_name.startswith('icon-art'): meteo_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + @@ -456,7 +470,7 @@ def main(cfg, model_cfg): #----------------------------------------------------- # In case of OEM: merge chem tracers with meteo-files #----------------------------------------------------- - if cfg.model == 'icon-art-oem': + if cfg.workflow_name == 'icon-art-oem': for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, cfg.meteo['inc']): @@ -531,6 +545,7 @@ def main(cfg, model_cfg): # If COSMO (and not ICON): else: + async_error(cfg, part='COSMO') logging.info('COSMO analysis data for IC/BC') dest_path = os.path.join(cfg.int2lm_input, 'meteo') @@ -625,7 +640,7 @@ def main(cfg, model_cfg): # Other IC/BC data inv_to_process = [] - if cfg.model == 'cosmo-ghg': + if cfg.workflow_name == 'cosmo-ghg': try: CAMS = dict(fullname="CAMS", nickname="cams", @@ -649,7 +664,7 @@ def main(cfg, model_cfg): inv_to_process.append(CT) except AttributeError: pass - elif cfg.model == 'cosmo-art': + elif cfg.workflow_name == 'cosmo-art': try: MOZART = dict(fullname='MOZART', nickname='mozart', @@ -664,7 +679,7 @@ def main(cfg, model_cfg): except AttributeError: pass - if cfg.model == 'cosmo-ghg' or cfg.model == 'cosmo-art': + if cfg.workflow_name == 'cosmo-ghg' or cfg.workflow_name == 'cosmo-art': logging.info("Processing " + ", ".join([i["fullname"] for i in inv_to_process]) + " data") diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index c0ef560a..ce652ffd 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -16,7 +16,7 @@ from . import tools -def main(startdate, enddate, cfg, model_cfg): +def main(startdate, enddate, cfg): """ Calculates 2D column data and writes them into a new netCDF file. Only a fixed number of levels from **COSMO** output are considered. diff --git a/jobs/tools/check_model.py b/jobs/tools/check_model.py index 9a1b29c1..5012d540 100644 --- a/jobs/tools/check_model.py +++ b/jobs/tools/check_model.py @@ -5,7 +5,7 @@ def check_model(cfg, model='COSMO'): """Check that the model specified in cfg matched the prescribed model. - Check that cfg.model == model. If not, raises a value-error. + Check that cfg.workflow_name == model. If not, raises a value-error. Ignores capitalization of the strings Parameters @@ -16,7 +16,7 @@ def check_model(cfg, model='COSMO'): Prescribed model """ #don't care about capitalization - if not cfg.model.lower() == model.lower(): + if not cfg.workflow_name.lower() == model.lower(): raise ValueError("The model specified in the configuration file is {}" ", but the job only applies to {}.".format( - cfg.model, model)) + cfg.workflow_name, model)) diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 61e31365..c188933a 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -19,7 +19,7 @@ def comp_data(dataset1, dataset2, variables): tools.helper.datasets_equal(dataset1, dataset2, variables, verbose=True) -def main(cfg, model_cfg): +def main(cfg): """Compare outputs of the chain to a reference. Looks for the reference-file in ``cfg.reference_dir``. diff --git a/run_chain.py b/run_chain.py index 743b54c2..7c5f1cb4 100755 --- a/run_chain.py +++ b/run_chain.py @@ -9,10 +9,10 @@ import time import shutil import argparse -import yaml import jobs from jobs import tools +from config import Config def parse_arguments(): @@ -44,7 +44,7 @@ def parse_arguments(): "Jobs are executed in the order in which they are " "given here. " "If no jobs are given, default jobs will be executed" - "as defined in config/models.yaml.") + "as defined in config/workflows.yaml.") parser.add_argument("-j", "--jobs", nargs='*', @@ -84,336 +84,7 @@ def parse_arguments(): return args -class Config(): - - def __init__(self, casename): - """Initialize an instance of the Config class. - - Initializes an instance of the Config class with user-specific - and default attributes. The class represents a processing chain for a - particular case, and its attributes are populated based on the provided - `casename`. - - Parameters - ---------- - casename : str - The identifier for the case, typically specifying the configuration - and settings to be used in the processing chain. - - Attributes - ---------- - user_name : str - The username of the current user, obtained from the 'USER' environment variable. - email : str - The user's email address, initially set to None and updated using the `set_email` method. - casename : str - The specified case name for the processing chain. - chain_src_dir : str - The source directory for the processing chain, typically the current working directory. - case_path : str - The path to the case directory under 'cases/' for the specified `casename`. - work_root : str - The root directory for processing chain execution, typically located under the source directory. - - Notes - ----- - The method also loads user-defined attributes from the configuration file, - sets specific settings based on the node type ('gpu' or 'mc'), and initializes - other instance-specific attributes. - """ - # Global attributes (initialized with default values) - self.user_name = os.environ['USER'] - self.set_email() - self.casename = casename - self.set_account() - - self.chain_src_dir = os.getcwd() - self.case_path = os.path.join(self.chain_src_dir, 'cases', - self.casename) - self.work_root = os.path.join(self.chain_src_dir, 'work') - - # User-defined attributes from config file - self.load_config_file(casename) - - # Specific settings based on the node type ('gpu' or 'mc') - self.set_node_info() - - def load_config_file(self, casename): - """Load configuration settings from a YAML file and set them as attributes. - - This method reads the configuration settings from a YAML file located in - the 'cases/casename' directory and sets them as attributes of the instance. - - Parameters - ---------- - casename : str - Name of the folder in 'cases/' where the configuration files are stored. - - Returns - ------- - Config - The same `Config` instance with configuration settings as attributes. - - Raises - ------ - FileNotFoundError - If the specified configuration file or case directory is not found. - - Notes - ----- - If the configuration file does not exist, the method will attempt to suggest - a similar case directory based on a Levenshtein distance comparison with - existing case directories. The method directly assigns values from the - configuration file to instance attributes for easy access. - """ - cfg_file = os.path.join('cases', casename, 'config.yaml') - - if not os.path.isfile(cfg_file): - all_cases = [ - path.name for path in os.scandir('cases') if path.is_dir() - ] - closest_name = min([(tools.levenshtein(casename, name), name) - for name in all_cases], - key=lambda x: x[0])[1] - raise FileNotFoundError( - f"Case-directory '{casename}' not found, did you mean '{closest_name}'?" - ) - - try: - with open(cfg_file, 'r') as yaml_file: - cfg_data = yaml.load(yaml_file, Loader=yaml.FullLoader) - except FileNotFoundError: - raise FileNotFoundError( - f"No file 'config.yaml' in {os.path.dirname(cfg_file)}") - - # Directly assign values to instance attributes - for key, value in cfg_data.items(): - setattr(self, key, value) - - return self - - def set_account(self): - """Set the compute account based on user information. - - This method determines the compute account to be used based on the user's - name and system configuration. - - Returns - ------- - Config - The same `Config` instance with the `compute_account` attribute set. - - Notes - ----- - - If the user name is 'jenkins', the compute account is set to 'g110' for - Jenkins testing. - - If an account is specified in the user's '~/.acct' file, it will be used - as the compute account. - - If neither of the above conditions is met, the standard account is - determined using the 'id -gn' command. - """ - if self.user_name == 'jenkins': - # g110 account for Jenkins testing - self.compute_account = 'g110' - elif os.path.exists(os.environ['HOME'] + '/.acct'): - # Use account specified in ~/.acct file - with open(os.environ['HOME'] + '/.acct', 'r') as file: - self.compute_account = file.read().rstrip() - else: - # Use standard account - self.compute_account = os.popen("id -gn").read().splitlines()[0] - - return self - - def set_node_info(self): - """Set node-specific information based on configuration settings. - - This method configures node-specific settings, such as the number of tasks - per node and CUDA-related environment variables, based on the provided - configuration settings in the instance. - - Returns - ------- - Config - The same `Config` instance with updated node-specific attributes. - - Raises - ------ - ValueError - If the 'constraint' or 'run_on' configuration values are invalid. - """ - if self.constraint == 'gpu': - if self.model.startswith('icon'): - if self.run_on == 'gpu': - self.ntasks_per_node = 1 - elif self.run_on == 'cpu': - self.ntasks_per_node = 12 - else: - raise ValueError( - "Invalid value for 'run_on' in the configuration." - "It should be either 'gpu' or 'cpu'.") - else: - self.ntasks_per_node = 12 - self.mpich_cuda = ('export MPICH_RDMA_ENABLED_CUDA=1\n' - 'export MPICH_G2G_PIPELINE=256\n' - 'export CRAY_CUDA_MPS=1\n') - elif self.constraint == 'mc': - self.ntasks_per_node = 36 - self.mpich_cuda = '' - else: - raise ValueError( - "Invalid value for 'constraint' in the configuration." - "It should be either 'gpu' or 'mc'.") - - return self - - def set_restart_step_hours(self): - """Set the restart step in hours. - - Converts the 'restart_step' attribute, which is in ISO8601 duration format, - to hours and stores the result in the 'restart_step_hours' attribute. - - Returns - ------- - Config - The same `Config` instance with the 'restart_step_hours' attribute set. - """ - self.restart_step_hours = int( - tools.iso8601_duration_to_hours(self.restart_step)) - - return self - - def set_email(self): - """Set the user's email address based on system configuration. - - This method determines the user's email address based on the user's name - and system configuration. - - Returns - ------- - Config - The same `Config` instance with the `user_mail` attribute set. - - Notes - ----- - - If the user name is 'jenkins', the user's email address is set to None. - - If an email address is specified in the user's '~/.forward' file, it will - be used as the user's email address. - - If neither of the above conditions is met, the user's email address is set - to None. - """ - if self.user_name == 'jenkins': - self.user_mail = None - elif os.path.exists(os.environ['HOME'] + '/.forward'): - with open(os.environ['HOME'] + '/.forward', 'r') as file: - self.user_mail = file.read().rstrip() - else: - self.user_mail = None - - return self - - def print_config(self): - """Print the configuration attributes and their values. - - This method displays the configuration attributes and their corresponding - values in a formatted manner. Lists and dictionaries within the configuration - are also displayed with appropriate indentation. - - Notes - ----- - - The maximum column width for the attribute names is automatically determined. - - The method prints the attribute name, its type, and its value. - - If an attribute is a list, it is displayed with each item indented. - - If an attribute is a dictionary, it is also displayed with each key-value - pair indented. - """ - # max_col_width = max(len(key) for key in vars(self)) + 1 - max_col_width = 27 - - print("\nConfiguration:") - print(f"{'Attribute':<{max_col_width}} Type Value") - print("-" * 80) - for key, value in vars(self).items(): - if isinstance(value, list): - # If the value is a list, format it with indentation - print(f"{key:<{max_col_width}} list") - for item in value: - item_type = type(item).__name__ - print(f" - {item:<{max_col_width-4}} {item_type}") - elif isinstance(value, dict): - # If the value is a dictionary, format it as before - print(f"{key:<{max_col_width}} dict") - for sub_key, sub_value in value.items(): - sub_value_type = type(sub_value).__name__ - print( - f" - {sub_key:<{max_col_width-4}} {sub_value_type:<4} {sub_value}" - ) - else: - # Standard output - key_type = type(key).__name__ - print(f"{key:<{max_col_width}} {key_type:<4} {value}") - - def convert_paths_to_absolute(self): - """Convert relative file paths to absolute paths in the configuration. - - This method iterates through all variables and their dictionary entries in - the configuration and checks for string values that represent file paths. - If a file path is relative (starts with './'), it is converted to an - absolute path using `os.path.abspath`. - - Returns - ------- - Config - The same `Config` instance with relative file paths converted to absolute paths. - """ - # Loop through all variables and their dictionary entries - for attr_name, attr_value in self.__dict__.items(): - if isinstance(attr_value, str): - if os.path.isabs(attr_value): - # If the value is already an absolute path, continue to the next iteration - continue - # Convert relative paths to absolute paths - if attr_value.startswith('./'): - self.__dict__[attr_name] = os.path.abspath(attr_value) - elif isinstance(attr_value, dict): - # If the attribute is a dictionary, loop through its entries - for key, value in attr_value.items(): - if isinstance(value, str): - if os.path.isabs(value): - # If the value is already an absolute path, continue to the next iteration - continue - # Convert relative paths to absolute paths - if value.startswith('./'): - self.__dict__[attr_name][key] = os.path.abspath( - value) - - return self - - def create_vars_from_dicts(self): - """Create instance attributes from dictionary entries in the configuration. - - This method iterates through the instance's attribute dictionary and checks - for dictionary values. For each dictionary encountered, it creates new - instance attributes by concatenating the original attribute name and the - dictionary key, and assigns the corresponding values. - - Returns - ------- - Config - The same `Config` instance with new attributes created from dictionary entries. - """ - # Create a copy of the object's __dict__ to avoid modifying it during iteration - object_dict = vars(self).copy() - - for key, value in object_dict.items(): - if isinstance(value, dict): - for sub_key, sub_value in value.items(): - setattr(self, key + '_' + sub_key, sub_value) - return self - - -def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, - force, resume): +def run_chain(cfg, force, resume): """Run the processing chain, managing job execution and logging. This function sets up and manages the execution of a processing chain, handling @@ -421,18 +92,8 @@ def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, Parameters ---------- - work_root : str - The path to the directory where the processing chain writes files during execution. - model_cfg : dict - Configuration settings for the modeling framework. cfg : Config Object holding user-defined configuration parameters as attributes. - startdate_sim : datetime-object - The start date of the simulation. - enddate_sim : datetime-object - The end date of the simulation. - job_names : list of str - List of names of jobs to execute on every timeslice. force : bool If True, it will force the execution of jobs regardless of their completion status. resume : bool @@ -448,22 +109,17 @@ def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, - This function sets various configuration values based on the provided parameters. - It checks for job completion status and resumes or forces execution accordingly. - Job log files are managed, and errors or timeouts are handled with notifications. - """ - # Write current start and end dates to config variables - cfg.startdate_sim = startdate_sim - cfg.enddate_sim = enddate_sim - - # Set forecast time + """# Set forecast time cfg.forecasttime = (cfg.enddate_sim - cfg.startdate_sim).total_seconds() / 3600 # String variables for startdate_sim - cfg.startdate_sim_yyyymmddhh = startdate_sim.strftime('%Y%m%d%H') - cfg.enddate_sim_yyyymmddhh = enddate_sim.strftime('%Y%m%d%H') + cfg.startdate_sim_yyyymmddhh = cfg.startdate_sim.strftime('%Y%m%d%H') + cfg.enddate_sim_yyyymmddhh = cfg.enddate_sim.strftime('%Y%m%d%H') # Folder naming and structure cfg.job_id = f'{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}' - cfg.chain_root = os.path.join(work_root, cfg.casename, cfg.job_id) + cfg.chain_root = cfg.work_root / cfg.casename / cfg.job_id # Config variables for spinup runs (datetimes, job-id, etc.) if hasattr(cfg, 'spinup'): @@ -486,15 +142,13 @@ def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, cfg.spinup)).strftime('%Y%m%d%H') cfg.job_id_prev = f'{startdate_sim_yyyymmddhh_prev}_{enddate_sim_yyyymmddhh_prev}' - cfg.chain_root_prev = os.path.join(work_root, cfg.casename, - cfg.job_id_prev) - cfg.last_cosmo_output = os.path.join(cfg.chain_root_prev, 'cosmo', - 'output') + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.job_id_prev + cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' # No restart for spinup simulations (= default values for no restart) cfg.cosmo_restart_out = '' cfg.cosmo_restart_in = '' - elif 'restart' in model_cfg['models'][cfg.model]['features']: + elif 'restart' in cfg.workflow['features']: cfg.startdate_sim_prev = cfg.startdate_sim - timedelta( hours=cfg.restart_step_hours) cfg.enddate_sim_prev = cfg.enddate_sim - timedelta( @@ -505,14 +159,11 @@ def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, '%Y%m%d%H') cfg.job_id_prev = f'{cfg.startdate_sim_prev_yyyymmddhh}_{cfg.enddate_sim_prev_yyyymmddhh}' - cfg.chain_root_prev = os.path.join(work_root, cfg.casename, - cfg.job_id_prev) + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.job_id_prev # Set restart directories - cfg.cosmo_restart_out = os.path.join(cfg.chain_root, 'cosmo', - 'restart') - cfg.cosmo_restart_in = os.path.join(cfg.chain_root_prev, 'cosmo', - 'restart') + cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' + cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' # Check constraint if hasattr(cfg, 'constraint'): @@ -520,21 +171,20 @@ def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, "gpu or mc") # If nested run: use output of mother-simulation - if 'nesting' in model_cfg['models'][ - cfg.model]['features'] and not os.path.isdir(cfg.meteo.dir): + if 'nesting' in cfg.workflow['features'] and not os.path.isdir( + cfg.meteo.dir): # if ifs_hres_dir doesn't point to a directory, # it is the name of the mother run mother_name = cfg.meteo.dir - cfg.meteo.dir = os.path.join(work_root, mother_name, cfg.job_id, - 'cosmo', 'output') + cfg.meteo.dir = cfg.work_root / mother_name / cfg.job_id / 'cosmo' / 'output' cfg.meteo.inc = 1 cfg.meteo.prefix = 'lffd' # Logging - log_working_dir = os.path.join(cfg.chain_root, 'checkpoints', 'working') - log_finished_dir = os.path.join(cfg.chain_root, 'checkpoints', 'finished') - setattr(cfg, 'log_working_dir', log_working_dir) - setattr(cfg, 'log_finished_dir', log_finished_dir) + log_working_dir = cfg.chain_root / 'checkpoints' / 'working' + log_finished_dir = cfg.chain_root / 'checkpoints' / 'finished' + cfg.log_working_dir = log_working_dir + cfg.log_finished_dir = log_finished_dir # Create working directories tools.create_dir(cfg.chain_root, "chain_root") @@ -543,84 +193,91 @@ def run_chain(work_root, model_cfg, cfg, startdate_sim, enddate_sim, job_names, # Number of levels and switch for unit conversion for 'reduce_output' job if not hasattr(cfg, 'output_levels'): - setattr(cfg, 'output_levels', -1) + cfg.output_levels = -1 if not hasattr(cfg, 'convert_gas'): - setattr(cfg, 'convert_gas', True) - - # run jobs (if required) - for job in job_names: - skip = False - - # if exists job is currently worked on or has been finished - if os.path.exists(os.path.join(log_working_dir, job)): - if not force: - while True: - if os.path.exists(os.path.join(log_finished_dir, job)): - print('Skip "%s" for chain "%s"' % (job, cfg.job_id)) - skip = True - break - elif resume: - resume = False - break - else: - print('Wait for "%s" of chain "%s"' % - (job, cfg.job_id)) - sys.stdout.flush() - for _ in range(3000): - time.sleep(0.1) - else: - os.remove(os.path.join(log_working_dir, job)) - try: - os.remove(os.path.join(log_finished_dir, job)) - except FileNotFoundError: - pass - - if not skip: - print('Process "%s" for chain "%s"' % (job, cfg.job_id)) - sys.stdout.flush() - - try_count = 1 + (args.ntry - 1) * (job == 'cosmo') - while try_count > 0: - try_count -= 1 - try: - # Change the log file - logfile = os.path.join(cfg.log_working_dir, job) - logfile_finish = os.path.join(cfg.log_finished_dir, job) - tools.change_logfile(logfile) - - # Launch the job - to_call = getattr(jobs, job) - to_call.main(cfg, model_cfg) - - shutil.copy(logfile, logfile_finish) - - exitcode = 0 - try_count = 0 - except: + cfg.convert_gas = True + + if cfg.is_async: + # Submit current chunck + # - [ ] This bypasses all the logfile moving/checking + # - [ ] Still needs a mechanism for resume + for job in cfg.jobs: + getattr(jobs, job).main(cfg) + + # wait for previsouy chunk to be done + cfg.wait_for_previous() + # cycle + cfg.job_ids['previous'] = cfg.job_ids['current'] + else: + # run jobs (if required) + for job in cfg.jobs: + skip = False + + # if exists job is currently worked on or has been finished + if (log_working_dir / job).exists(): + if not force: + while True: + if (log_finished_dir / job).exists(): + print(f"Skip {job} for chain {cfg.job_id}") + skip = True + break + elif resume: + resume = False + break + else: + print(f"Wait for {job} of chain {cfg.job_id}") + sys.stdout.flush() + for _ in range(3000): + time.sleep(0.1) + else: + (log_working_dir / job).unlink() + (log_finished_dir / job).unlink(missing_ok=True) + + if not skip: + print('Process "%s" for chain "%s"' % (job, cfg.job_id)) + sys.stdout.flush() + + try_count = 1 + (args.ntry - 1) * (job == 'cosmo') + while try_count > 0: + try_count -= 1 + try: + # Change the log file + logfile = cfg.log_working_dir / job + logfile_finish = cfg.log_finished_dir / job + tools.change_logfile(logfile) + + # Launch the job + to_call = getattr(jobs, job) + to_call.main(cfg) + + shutil.copy(logfile, logfile_finish) + + exitcode = 0 + try_count = 0 + except: + subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( + job, cfg.job_id) + logging.exception(subject) + if cfg.user_mail: + message = tools.prepare_message(log_working_dir / + job) + logging.info('Sending log file to %s' % + cfg.user_mail) + tools.send_mail(cfg.user_mail, subject, message) + if try_count == 0: + raise RuntimeError(subject) + + if exitcode != 0 or not (log_finished_dir / job).exists(): subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( job, cfg.job_id) - logging.exception(subject) if cfg.user_mail: - message = tools.prepare_message( - os.path.join(log_working_dir, job)) + message = tools.prepare_message(log_working_dir / job) logging.info('Sending log file to %s' % cfg.user_mail) tools.send_mail(cfg.user_mail, subject, message) - if try_count == 0: - raise RuntimeError(subject) - - if exitcode != 0 or not os.path.exists( - os.path.join(log_finished_dir, job)): - subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( - job, cfg.job_id) - if cfg.user_mail: - message = tools.prepare_message( - os.path.join(log_working_dir, job)) - logging.info('Sending log file to %s' % cfg.user_mail) - tools.send_mail(cfg.user_mail, subject, message) - raise RuntimeError(subject) - - -def restart_runs(work_root, model_cfg, cfg, job_names, force, resume): + raise RuntimeError(subject) + + +def restart_runs(cfg, force, resume): """Start subchains in specified intervals and manage restarts. This function slices the total runtime of the processing chain according to the @@ -629,14 +286,8 @@ def restart_runs(work_root, model_cfg, cfg, job_names, force, resume): Parameters ---------- - work_root : str - The path to the directory in which the chain writes files during execution. - model_cfg : dict - Configuration settings for the modeling framework. cfg : Config Object holding all user-configuration parameters as attributes. - job_names : list of str - List of names of jobs to execute on every timeslice. force : bool If True, it will force the execution of jobs regardless of their completion status. resume : bool @@ -656,24 +307,19 @@ def restart_runs(work_root, model_cfg, cfg, job_names, force, resume): continue # Set restart variable (only takes effect for ICON) - if startdate_sim == cfg.startdate: - setattr(cfg, "lrestart", '.FALSE.') - else: - setattr(cfg, "lrestart", '.TRUE.') + cfg.lrestart = '.FALSE.' if startdate_sim == cfg.startdate else '.TRUE.' + + print(f"Starting run with startdate {startdate_sim}") - print("Starting run with startdate {}".format(startdate_sim)) + cfg.startdate_sim = startdate_sim + cfg.enddate_sim = enddate_sim - run_chain(work_root=work_root, - model_cfg=model_cfg, - cfg=cfg, - startdate_sim=startdate_sim, - enddate_sim=enddate_sim, - job_names=job_names, + run_chain(cfg=cfg force=force, resume=resume) -def restart_runs_spinup(work_root, model_cfg, cfg, job_names, force, resume): +def restart_runs_spinup(cfg, force, resume): """Start subchains in specified intervals and manage restarts with spin-up. This function slices the total runtime of the processing chain according to the @@ -682,13 +328,7 @@ def restart_runs_spinup(work_root, model_cfg, cfg, job_names, force, resume): Parameters ---------- - work_root : str - The path to the directory in which the chain writes files during execution. - model_cfg : dict - Configuration settings for the modeling framework. cfg : Config - Object holding all user-configuration parameters as attributes. - job_names : list of str List of names of jobs to execute on every timeslice. force : bool If True, it will force the execution of jobs regardless of their completion status. @@ -703,22 +343,22 @@ def restart_runs_spinup(work_root, model_cfg, cfg, job_names, force, resume): for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, cfg.restart_step_hours): if startdate_sim == cfg.startdate: - setattr(cfg, "first_one", True) - setattr(cfg, "second_one", False) - setattr(cfg, "lrestart", '.FALSE.') + cfg.first_one = True + cfg.second_one = False + cfg.lrestart = '.FALSE.' run_time = cfg.restart_step_hours startdate_sim_spinup = startdate_sim elif startdate_sim == cfg.startdate + timedelta( hours=cfg.restart_step_hours): - setattr(cfg, "first_one", False) - setattr(cfg, "second_one", True) - setattr(cfg, "lrestart", '.TRUE.') + cfg.first_one = False + cfg.second_one = True + cfg.lrestart = '.TRUE.' run_time = cfg.restart_step_hours + cfg.spinup startdate_sim_spinup = startdate_sim - timedelta(hours=cfg.spinup) else: - setattr(cfg, "first_one", False) - setattr(cfg, "second_one", False) - setattr(cfg, "lrestart", '.TRUE.') + cfg.first_one = False + cfg.second_one = False + cfg.lrestart = '.TRUE.' run_time = cfg.restart_step_hours + cfg.spinup startdate_sim_spinup = startdate_sim - timedelta(hours=cfg.spinup) @@ -729,35 +369,15 @@ def restart_runs_spinup(work_root, model_cfg, cfg, job_names, force, resume): print(f'Runtime of sub-simulation: {run_time} h') - run_chain(work_root=work_root, - model_cfg=model_cfg, - cfg=cfg, - startdate_sim=startdate_sim_spinup, - enddate_sim=enddate_sim, - job_names=job_names, + cfg.startdate_sim = startdate_sim_spinup + cfg.enddate_sim = enddate_sim + + run_chain(cfg=cfg, force=force, resume=resume) -def load_model_config_yaml(yamlfile): - """Load model configuration from a YAML file. - - Parameters - ---------- - yamlfile : str - The path to the YAML file containing the model configuration. - - Returns - ------- - dict - A dictionary representing the model configuration loaded from the YAML file. - """ - with open(yamlfile) as file: - model_cfg = yaml.safe_load(file) - return model_cfg - - -if __name__ == '__main__': +def main(): """Main script for running a processing chain. This script handles the execution of a processing chain for one or more specified cases. It loads model configurations, prepares the environment, and starts the chain based on the provided settings. @@ -776,7 +396,6 @@ def load_model_config_yaml(yamlfile): for casename in args.casenames: # Load configs - model_cfg = load_model_config_yaml('config/models.yaml') cfg = Config(casename) # Convert relative to absolute paths @@ -795,36 +414,36 @@ def load_model_config_yaml(yamlfile): # Check if jobs are set or if default ones are used if args.job_list is None: - args.job_list = model_cfg['models'][cfg.model]['jobs'] + cfg.jobs = cfg.workflow['jobs'] + else: + cfg.jobs = args.job_list - print(f"Starting chain for case {casename} and model {cfg.model}") + print( + f"Starting chain for case {casename} and workflow {cfg.workflow_name}" + ) # Check for restart compatibility and spinup - if 'restart' in model_cfg['models'][cfg.model]['features']: + if 'restart' in cfg.workflow['features']: if hasattr(cfg, 'spinup'): print("Using spin-up restarts.") - restart_runs_spinup(work_root=cfg.work_root, - model_cfg=model_cfg, - cfg=cfg, - job_names=args.job_list, + restart_runs_spinup(cfg=cfg, force=args.force, resume=args.resume) else: print("Using built-in model restarts.") - restart_runs(work_root=cfg.work_root, - model_cfg=model_cfg, - cfg=cfg, - job_names=args.job_list, + restart_runs(cfg=cfg, force=args.force, resume=args.resume) else: print("No restarts are used.") - run_chain(work_root=cfg.work_root, - cfg=cfg, - startdate_sim=cfg.startdate, - enddate_sim=cfg.enddate, - job_names=args.job_list, + cfg.startdate_sim = cfg.startdate + cfg.enddate_sim = cfg.enddate + run_chain(cfg=cfg, force=args.force, resume=args.resume) print('>>> Finished the processing chain successfully <<<') + + +if __name__ == '__main__': + main() diff --git a/workflows.yaml b/workflows.yaml new file mode 100644 index 00000000..bcbc56f4 --- /dev/null +++ b/workflows.yaml @@ -0,0 +1,27 @@ +cosmo: + jobs: ['prepare_data', 'int2lm', 'cosmo', 'post_cosmo'] + features: ['restart', 'spinup'] +cosmo-ghg: + jobs: ['prepare_data', 'emissions', 'biofluxes', 'oem', 'online_vprm', + 'int2lm', 'post_int2lm', 'cosmo', 'post_cosmo'] + features: ['restart', 'tracers', 'spinup'] +cosmo-art: + jobs: ['prepare_data', 'emissions', 'obs_nudging', 'photo_rate', + 'int2lm', 'cosmo', 'post_cosmo'] + features: ['nesting', 'spinup'] +icon: + jobs: ['prepare_data', 'icon'] + features: ['restart'] + dependencies: + icon: + current: ['prepare_data'] + previous: ['icon'] +icon-art: + jobs: ['prepare_data', 'icon'] + features: ['restart'] +icon-art-global: + jobs: ['prepare_data', 'icon'] + features: ['restart'] +icon-art-oem: + jobs: ['prepare_data', 'icon'] + features: ['restart'] From fedc316a76353ae6b73aed727f61c87e47f1027c Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 12 Dec 2023 13:27:49 +0000 Subject: [PATCH 002/245] GitHub Action: Apply Pep8-formatting --- config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 299f7220..3cfc35eb 100644 --- a/config.py +++ b/config.py @@ -377,14 +377,16 @@ def submit(self, job_name, script, add_dep=None): if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): sbatch_cmd.append(dep_cmd) sbatch_cmd.append(script_path.name) - - result = subprocess.run(sbatch_cmd, cwd=script_path.parent, capture_output=True) + + result = subprocess.run(sbatch_cmd, + cwd=script_path.parent, + capture_output=True) job_id = int(result.stdout) if not job_name in self.job_ids['current']: self.job_ids['current'][job_name] = [job_id] else: self.job_ids['current'][job_name].append(job_id) - + # If needed internaly in a multi-job task like prepare_data # Can then be passed as add_dep keyword return job_id From 0d1c5b7f2cddc6567abc4d787b1b7e8a93bde8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 12 Dec 2023 15:24:41 +0100 Subject: [PATCH 003/245] Update gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index efb2d3f0..b2df1de5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,6 @@ input_processing-chain.tgz input/ output/ work/ -src/*/ +ext/*/ *.code-workspace -.vscode/ \ No newline at end of file +.vscode/ From 2168db93944a3b3e77702099957f0a902a9db3a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 12 Dec 2023 16:22:23 +0100 Subject: [PATCH 004/245] Fix syntax error --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 7c5f1cb4..f331339f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -314,7 +314,7 @@ def restart_runs(cfg, force, resume): cfg.startdate_sim = startdate_sim cfg.enddate_sim = enddate_sim - run_chain(cfg=cfg + run_chain(cfg=cfg, force=force, resume=resume) From 4f80aba0bea22839ead264778d4383802d21bb47 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 12 Dec 2023 15:22:50 +0000 Subject: [PATCH 005/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 21 ++++++++++----------- run_chain.py | 16 ++++------------ 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 9d468349..359ecba6 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -62,7 +62,8 @@ def set_cfg_variables(cfg): def async_error(cfg, part="This part"): if cfg.is_async: - raise NotImplementedError(f"{part} isn't ready for async execution yet") + raise NotImplementedError( + f"{part} isn't ready for async execution yet") def main(cfg): @@ -120,20 +121,18 @@ def main(cfg): #----------------------------------------------------- wall_time = getattr(cfg, 'copy_input_walltime', '00:01:00') queue = getattr(cfg, 'copy_input_queue', 'normal') - - script_lines = ['#!/usr/bin/env bash', - f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', - f'#SBATCH --account={cfg.compute_account}', - f'#SBATCH --time={walltime}', - f'#SBATCH --partition={queue}', - '#SBATCH --constraint=gpu', - '#SBATCH --nodes=1', - ''] + + script_lines = [ + '#!/usr/bin/env bash', + f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', + f'#SBATCH --account={cfg.compute_account}', + f'#SBATCH --time={walltime}', f'#SBATCH --partition={queue}', + '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', '' + ] for target, destination in zip(cfg.input_files.values(), cfg.input_files_scratch.values()): script_lines.append(f'rsync -av {target} {destination}') - with (script := cfg.icon_base / 'copy_input.job').open('w') as f: f.write('\n'.join(script_lines)) diff --git a/run_chain.py b/run_chain.py index f331339f..f88d02bb 100755 --- a/run_chain.py +++ b/run_chain.py @@ -314,9 +314,7 @@ def restart_runs(cfg, force, resume): cfg.startdate_sim = startdate_sim cfg.enddate_sim = enddate_sim - run_chain(cfg=cfg, - force=force, - resume=resume) + run_chain(cfg=cfg, force=force, resume=resume) def restart_runs_spinup(cfg, force, resume): @@ -372,9 +370,7 @@ def restart_runs_spinup(cfg, force, resume): cfg.startdate_sim = startdate_sim_spinup cfg.enddate_sim = enddate_sim - run_chain(cfg=cfg, - force=force, - resume=resume) + run_chain(cfg=cfg, force=force, resume=resume) def main(): @@ -431,16 +427,12 @@ def main(): resume=args.resume) else: print("Using built-in model restarts.") - restart_runs(cfg=cfg, - force=args.force, - resume=args.resume) + restart_runs(cfg=cfg, force=args.force, resume=args.resume) else: print("No restarts are used.") cfg.startdate_sim = cfg.startdate cfg.enddate_sim = cfg.enddate - run_chain(cfg=cfg, - force=args.force, - resume=args.resume) + run_chain(cfg=cfg, force=args.force, resume=args.resume) print('>>> Finished the processing chain successfully <<<') From c89f35ccce6d4b4442b6c0d7213d16c79b2baa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 13 Dec 2023 14:30:30 +0100 Subject: [PATCH 006/245] Add missing imports --- config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.py b/config.py index 3cfc35eb..59c89100 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,7 @@ import subprocess import os +import yaml +from jobs import tools from pathlib import Path From c6e9bfae5c6b2e8173598789ef77182c05048c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 13 Dec 2023 16:01:32 +0100 Subject: [PATCH 007/245] Fix remaining syntax errors --- config.py | 2 +- run_chain.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 59c89100..70bd1abf 100644 --- a/config.py +++ b/config.py @@ -361,7 +361,7 @@ def get_dep_cmd(self, job_name, add_dep=None): if self.is_async: # async case if dep_ids := self.get_dep_ids(job_name, add_dep=add_dep): - dep_str = ':'.join(map(str, deps_ids)) + dep_str = ':'.join(map(str, dep_ids)) return f'--dependency=afterok:{dep_str}' else: # job_name has no dependencies but still belongs to an async workflow diff --git a/run_chain.py b/run_chain.py index f88d02bb..7eb6c96a 100755 --- a/run_chain.py +++ b/run_chain.py @@ -109,7 +109,8 @@ def run_chain(cfg, force, resume): - This function sets various configuration values based on the provided parameters. - It checks for job completion status and resumes or forces execution accordingly. - Job log files are managed, and errors or timeouts are handled with notifications. - """# Set forecast time + """ + # Set forecast time cfg.forecasttime = (cfg.enddate_sim - cfg.startdate_sim).total_seconds() / 3600 @@ -237,7 +238,7 @@ def run_chain(cfg, force, resume): print('Process "%s" for chain "%s"' % (job, cfg.job_id)) sys.stdout.flush() - try_count = 1 + (args.ntry - 1) * (job == 'cosmo') + try_count = 1 + (cfg.ntry - 1) * (job == 'cosmo') while try_count > 0: try_count -= 1 try: @@ -376,7 +377,9 @@ def restart_runs_spinup(cfg, force, resume): def main(): """Main script for running a processing chain. - This script handles the execution of a processing chain for one or more specified cases. It loads model configurations, prepares the environment, and starts the chain based on the provided settings. + This script handles the execution of a processing chain for one or more + specified cases. It loads model configurations, prepares the environment, + and starts the chain based on the provided settings. Parameters ---------- @@ -385,8 +388,10 @@ def main(): Notes ----- - This script uses command-line arguments to specify cases and job lists. - - It loads model configurations, converts paths to absolute, sets restart settings, and starts the chain. - - Depending on the model's features, it may run with or without restarts or utilize spin-up restarts. + - It loads model configurations, converts paths to absolute, sets restart + settings, and starts the chain. + - Depending on the model's features, it may run with or without restarts + or utilize spin-up restarts. """ args = parse_arguments() @@ -394,6 +399,9 @@ def main(): # Load configs cfg = Config(casename) + # Make ntry a Config variable + cfg.ntry = args.ntry + # Convert relative to absolute paths cfg.convert_paths_to_absolute() From 05ef0dc426aa7668e183108b522d577d23b057c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 13 Dec 2023 16:29:13 +0100 Subject: [PATCH 008/245] Remove model_cfg from all jobs --- jobs/biofluxes.py | 4 +--- jobs/check_output.py | 4 +--- jobs/cosmo.py | 4 +--- jobs/emissions.py | 4 +--- jobs/icon.py | 4 +--- jobs/int2lm.py | 4 +--- jobs/obs_nudging.py | 2 -- jobs/octe.py | 2 -- jobs/oem.py | 2 -- jobs/online_vprm.py | 4 +--- jobs/photo_rate.py | 2 -- jobs/post_cosmo.py | 2 -- jobs/post_int2lm.py | 2 -- jobs/prepare_data.py | 4 +--- jobs/reduce_output.py | 4 +--- jobs/verify_chain.py | 2 -- 16 files changed, 9 insertions(+), 41 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index b494323e..2834f4ef 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -6,7 +6,7 @@ from . import tools, prepare_data -def main(cfg, model_cfg): +def main(cfg): """Prepare biofluxes files for COSMO simulations. Copies biofluxes files from the project folder (:attr:`cfg.vprm['dir']`) @@ -16,8 +16,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ tools.check_model(cfg, 'cosmo-ghg') cfg = prepare_data.set_cfg_variables(cfg) diff --git a/jobs/check_output.py b/jobs/check_output.py index add736c8..ccf45595 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -699,7 +699,7 @@ def create_animations(cfg): duration=300) -def main(cfg, model_cfg): +def main(cfg): """Check output variables for physical reasonability and create plots. This function checks the output variables to ensure they are in a physically @@ -712,8 +712,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ date = dt.datetime.today() diff --git a/jobs/cosmo.py b/jobs/cosmo.py index d21f0a37..a74155d1 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -52,7 +52,7 @@ def set_cfg_variables(cfg): return cfg -def main(cfg, model_cfg): +def main(cfg): """Setup the namelists for a COSMO tracer run and submit the job to the queue. Necessary for both COSMO and COSMOART simulations. @@ -80,8 +80,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ cfg = set_cfg_variables(cfg) logfile = os.path.join(cfg.log_working_dir, "cosmo") diff --git a/jobs/emissions.py b/jobs/emissions.py index 7384e3dc..83ef94b9 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -7,7 +7,7 @@ from . import tools, prepare_data -def main(cfg, model_cfg): +def main(cfg): """Copy emission files to the int2lm input directory. Necessary for both COSMO and COSMOART simulations. @@ -29,8 +29,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ cfg = prepare_data.set_cfg_variables(cfg) dest_prefix = "emis_" diff --git a/jobs/icon.py b/jobs/icon.py index be93aca9..6d1943ff 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -7,7 +7,7 @@ from . import tools, prepare_data -def main(cfg, model_cfg): +def main(cfg): """Setup the namelists for an ICON tracer run and submit the job to the queue. @@ -32,8 +32,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ cfg = prepare_data.set_cfg_variables(cfg) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 44e6e5bf..4841864e 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -19,7 +19,7 @@ def set_cfg_variables(cfg): return cfg -def main(cfg, model_cfg): +def main(cfg): """Setup the namelist for int2lm and submit the job to the queue. Necessary for both COSMO and COSMOART simulations. @@ -49,8 +49,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ cfg = prepare_data.set_cfg_variables(cfg) cfg = set_cfg_variables(cfg) diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index a31371f1..b5539728 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -26,8 +26,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ dest_dir = os.path.join(cfg.cosmo_input, "obs_nudging") tools.create_dir(dest_dir, "obs nudging input") diff --git a/jobs/octe.py b/jobs/octe.py index 75f6d97c..f95193e1 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -176,8 +176,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ dest_dir = join(cfg.cosmo_input, 'octe') create_dir_and_copy_input(dest_dir=dest_dir, diff --git a/jobs/oem.py b/jobs/oem.py index fa9b8c74..a6e708f6 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -15,8 +15,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. Raises ------ diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index c87bac6b..740b92b1 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -7,7 +7,7 @@ from . import tools -def main(cfg, model_cfg): +def main(cfg): """Copy MODIS surface reflectance data and vegetation class fraction file to the **cosmo** input directory. @@ -15,8 +15,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ dest_modis = 'modis.nc' dest_vegetation = 'vegetation.nc' diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index 4cfcb817..72536394 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -19,8 +19,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ tools.check_model(cfg, 'cosmo-art') diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index f1119636..b3002025 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -62,8 +62,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ cfg = int2lm.set_cfg_variables(cfg) cfg = cosmo.set_cfg_variables(cfg) diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 4a93763e..936e1ac1 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -27,8 +27,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config/models.yaml`` file. """ cfg = int2lm.set_cfg_variables(cfg) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 359ecba6..b5740fc1 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -95,15 +95,13 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes. - model_cfg : dict - Model configuration settings loaded from the ``config\/models.yaml`` file. Raises ------ RuntimeError If any subprocess returns a non-zero exit code during execution. """ - cfg = set_cfg_variables(cfg, model_cfg) + cfg = set_cfg_variables(cfg) if cfg.workflow_name.startswith('icon'): logging.info('ICON input data (IC/BC)') diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index a1f8e57e..8736df2b 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -16,7 +16,7 @@ from . import tools -def main(cfg, model_cfg): +def main(cfg): """ Calculates 2D column data and writes them into a new netCDF file. Only a fixed number of levels from **COSMO** output are considered. @@ -41,8 +41,6 @@ def main(cfg, model_cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes - model_cfg : dict - Model configuration settings loaded from the ``config\/models.yaml`` file. """ cosmo_output = cfg.cosmo_output output_path = cfg.cosmo_output_reduced diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index ae79f049..45aa4eab 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -37,8 +37,6 @@ def main(cfg): ---------- cfg : Config Object holding all user-configuration parameters as attributes - model_cfg : dict - Model configuration settings loaded from the ``config\/models.yaml`` file. """ logging.info("Started verification") for (ref_file, From 811e826b0eafe3901a04e87e9e390bb62570ccf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 13 Dec 2023 16:46:10 +0100 Subject: [PATCH 009/245] Format print config --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 70bd1abf..1ef843f9 100644 --- a/config.py +++ b/config.py @@ -267,13 +267,13 @@ def print_config(self): # If the value is a list, format it with indentation print(f"{key:<{max_col_width}} list") for item in value: - item_type = type(item).__name__ + item_type = "Path" if type(item).__name__ == "PosixPath" else type(item).__name__ print(f" - {item:<{max_col_width-4}} {item_type}") elif isinstance(value, dict): # If the value is a dictionary, format it as before print(f"{key:<{max_col_width}} dict") for sub_key, sub_value in value.items(): - sub_value_type = type(sub_value).__name__ + sub_value_type = "Path" if type(sub_value).__name__ == "PosixPath" else type(sub_value).__name__ print( f" - {sub_key:<{max_col_width-4}} {sub_value_type:<4} {sub_value}" ) From 1d6c32a38990af38385a0d9b5921435d33846188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 13 Dec 2023 17:01:39 +0100 Subject: [PATCH 010/245] Fix jobs for cosmo workflow --- cases/icon-test/config.yaml | 3 +++ jobs/biofluxes.py | 2 +- jobs/cosmo.py | 39 ++++++++++++++++--------------------- jobs/emissions.py | 2 +- jobs/int2lm.py | 12 ++++-------- jobs/post_cosmo.py | 5 ++--- jobs/post_int2lm.py | 2 +- jobs/prepare_data.py | 10 ++-------- 8 files changed, 31 insertions(+), 44 deletions(-) diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index c4f79d0c..bbdbd9be 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -40,6 +40,9 @@ input_files: lrtm_filename: ./input/icon/rad/rrtmg_lw.nc map_file_ana: ./input/icon/mapping/map_file.ana +prepare_data: + walltime: '00:10:00' + icon: binary_file: ./ext/icon/bin/icon runjob_filename: icon_runjob.cfg diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 2834f4ef..b33d1524 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -18,7 +18,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.check_model(cfg, 'cosmo-ghg') - cfg = prepare_data.set_cfg_variables(cfg) + prepare_data.set_cfg_variables(cfg) scratch_path = os.path.join(cfg.int2lm_input, 'vprm') diff --git a/jobs/cosmo.py b/jobs/cosmo.py index a74155d1..c9552208 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -11,45 +11,40 @@ def set_cfg_variables(cfg): - setattr(cfg, 'cosmo_base', os.path.join(cfg.chain_root, 'cosmo')) - setattr(cfg, 'cosmo_input', os.path.join(cfg.chain_root, 'cosmo', 'input')) - setattr(cfg, 'cosmo_run', os.path.join(cfg.chain_root, 'cosmo', 'run')) - setattr(cfg, 'cosmo_output', os.path.join(cfg.chain_root, 'cosmo', - 'output')) - setattr(cfg, 'cosmo_output_reduced', - os.path.join(cfg.chain_root, 'cosmo', 'output_reduced')) + cfg.cosmo_base = cfg.chain_root / 'cosmo' + cfg.cosmo_input = cfg.chain_root / 'cosmo' / 'input' + cfg.cosmo_run = cfg.chain_root / 'cosmo' / 'run' + cfg.cosmo_output = cfg.chain_root / 'cosmo' / 'output' + cfg.cosmo_output_reduced = cfg.chain_root / 'cosmo' / 'output_reduced' # Number of tracers if 'tracers' in cfg.workflow['features']: - tracer_csvfile = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, - 'cosmo_tracers.csv') - if os.path.isfile(tracer_csvfile): + tracer_csvfile = cfg.chain_src_dir / 'cases' / cfg.casename / 'cosmo_tracers.csv' + if tracer_csvfile.is_file(): with open(tracer_csvfile, 'r') as csv_file: reader = csv.DictReader(csv_file, delimiter=',') reader = [r for r in reader if r[''] != '#'] - setattr(cfg, 'in_tracers', len(reader)) + cfg.in_tracers = len(reader) else: raise FileNotFoundError(f"File not found: {tracer_csvfile}") - # tracer_start namelist paramter for spinup simulation + # tracer_start namelist parameter for spinup simulation if hasattr(cfg, 'spinup'): if cfg.first_one: - setattr(cfg, 'tracer_start', 0) + cfg.tracer_start = 0 else: - setattr(cfg, 'tracer_start', cfg.spinup) + cfg.tracer_start = cfg.spinup else: - setattr(cfg, 'tracer_start', 0) + cfg.tracer_start = 0 # asynchronous I/O if hasattr(cfg, 'cfg.cosmo_np_io'): if cfg.cosmo_np_io == 0: - setattr(cfg, 'lasync_io', '.FALSE.') - setattr(cfg, 'num_iope_percomm', 0) + cfg.lasync_io = '.FALSE.' + cfg.num_iope_percomm = 0 else: - setattr(cfg, 'lasync_io', '.TRUE.') - setattr(cfg, 'num_iope_percomm', 1) - - return cfg + cfg.lasync_io = '.TRUE.' + cfg.num_iope_percomm = 1 def main(cfg): @@ -81,7 +76,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - cfg = set_cfg_variables(cfg) + set_cfg_variables(cfg) logfile = os.path.join(cfg.log_working_dir, "cosmo") logfile_finish = os.path.join(cfg.log_finished_dir, "cosmo") diff --git a/jobs/emissions.py b/jobs/emissions.py index 83ef94b9..a4cdf207 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -30,7 +30,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - cfg = prepare_data.set_cfg_variables(cfg) + prepare_data.set_cfg_variables(cfg) dest_prefix = "emis_" if not isinstance(cfg.emissions['dir'], list): diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 4841864e..c6b98fdc 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -11,12 +11,8 @@ def set_cfg_variables(cfg): - - setattr(cfg, 'int2lm_run', os.path.join(cfg.chain_root, 'int2lm', 'run')) - setattr(cfg, 'int2lm_output', - os.path.join(cfg.chain_root, 'int2lm', 'output')) - - return cfg + cfg.int2lm_run = cfg.chain_root / 'int2lm' / 'run' + cfg.int2lm_output = cfg.chain_root / 'int2lm' / 'output' def main(cfg): @@ -50,8 +46,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - cfg = prepare_data.set_cfg_variables(cfg) - cfg = set_cfg_variables(cfg) + prepare_data.set_cfg_variables(cfg) + set_cfg_variables(cfg) # Total number of processes np_tot = cfg.int2lm['np_x'] * cfg.int2lm['np_y'] diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index b3002025..309f2745 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -63,11 +63,10 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - cfg = int2lm.set_cfg_variables(cfg) - cfg = cosmo.set_cfg_variables(cfg) + int2lm.set_cfg_variables(cfg) + cosmo.set_cfg_variables(cfg) logfile = os.path.join(cfg.log_working_dir, "post_cosmo") - cosmo_run_dir = cfg.cosmo_run runscript_path = os.path.join(cfg.cosmo_run, "post_cosmo.job") copy_path = os.path.join( cfg.post_cosmo['output_root'], diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 936e1ac1..129115b7 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -28,7 +28,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - cfg = int2lm.set_cfg_variables(cfg) + int2lm.set_cfg_variables(cfg) # Int2lm processing always starts at hstart=0, thus modifying inidate inidate_int2lm_yyyymmddhh = cfg.startdate_sim_yyyymmddhh diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index b5740fc1..dae57f87 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -101,7 +101,7 @@ def main(cfg): RuntimeError If any subprocess returns a non-zero exit code during execution. """ - cfg = set_cfg_variables(cfg) + set_cfg_variables(cfg) if cfg.workflow_name.startswith('icon'): logging.info('ICON input data (IC/BC)') @@ -114,17 +114,11 @@ def main(cfg): tools.create_dir(cfg.icon_output, "icon_output") tools.create_dir(cfg.icon_restart_out, "icon_restart_out") - #----------------------------------------------------- - # Copy input files - #----------------------------------------------------- - wall_time = getattr(cfg, 'copy_input_walltime', '00:01:00') - queue = getattr(cfg, 'copy_input_queue', 'normal') - script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', f'#SBATCH --account={cfg.compute_account}', - f'#SBATCH --time={walltime}', f'#SBATCH --partition={queue}', + f'#SBATCH --time={cfg.prepare_data_walltime}', f'#SBATCH --partition={cfg.compute_queue}', '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', '' ] for target, destination in zip(cfg.input_files.values(), From 2cb97d11a28be021a5ec7c7e2aa707a320d4cf4c Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 13 Dec 2023 16:02:08 +0000 Subject: [PATCH 011/245] GitHub Action: Apply Pep8-formatting --- config.py | 7 +++++-- jobs/prepare_data.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 1ef843f9..f35cf142 100644 --- a/config.py +++ b/config.py @@ -267,13 +267,16 @@ def print_config(self): # If the value is a list, format it with indentation print(f"{key:<{max_col_width}} list") for item in value: - item_type = "Path" if type(item).__name__ == "PosixPath" else type(item).__name__ + item_type = "Path" if type( + item).__name__ == "PosixPath" else type(item).__name__ print(f" - {item:<{max_col_width-4}} {item_type}") elif isinstance(value, dict): # If the value is a dictionary, format it as before print(f"{key:<{max_col_width}} dict") for sub_key, sub_value in value.items(): - sub_value_type = "Path" if type(sub_value).__name__ == "PosixPath" else type(sub_value).__name__ + sub_value_type = "Path" if type( + sub_value).__name__ == "PosixPath" else type( + sub_value).__name__ print( f" - {sub_key:<{max_col_width-4}} {sub_value_type:<4} {sub_value}" ) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index dae57f87..3738d45f 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -118,7 +118,8 @@ def main(cfg): '#!/usr/bin/env bash', f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', f'#SBATCH --account={cfg.compute_account}', - f'#SBATCH --time={cfg.prepare_data_walltime}', f'#SBATCH --partition={cfg.compute_queue}', + f'#SBATCH --time={cfg.prepare_data_walltime}', + f'#SBATCH --partition={cfg.compute_queue}', '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', '' ] for target, destination in zip(cfg.input_files.values(), From e605144eac16eaafb122158b9c10af6b9002b39b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 13 Dec 2023 17:31:40 +0100 Subject: [PATCH 012/245] Further fixes --- jobs/oem.py | 2 +- jobs/post_cosmo.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jobs/oem.py b/jobs/oem.py index a6e708f6..4217b48e 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -21,7 +21,7 @@ def main(cfg): RuntimeError If an error occurs during the process. """ - cfg = cosmo.set_cfg_variables(cfg) + cosmo.set_cfg_variables(cfg) oem_dir = cfg.oem['dir'] oem_gridded_emissions_nc = os.path.join(oem_dir, diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 309f2745..72dae422 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -84,11 +84,11 @@ def main(cfg): cosmo_run=cfg.cosmo_run) if os.path.isdir(cfg.cosmo_output_reduced): - cosmo_output_src = cfg.cosmo_output_reduced.rstrip('/') + cosmo_output_src = str(cfg.cosmo_output_reduced).rstrip('/') cosmo_output_dest = os.path.join(copy_path, "cosmo_output_reduced").rstrip('/') else: - cosmo_output_src = cfg.cosmo_output.rstrip('/') + cosmo_output_src = str(cfg.cosmo_output).rstrip('/') cosmo_output_dest = os.path.join(copy_path, "cosmo_output").rstrip('/') # Create new directories @@ -105,13 +105,13 @@ def main(cfg): # Format the runscript runscript_content += runscript_commands_template().format( target_dir=copy_path.rstrip('/'), - int2lm_run_src=cfg.int2lm_run.rstrip('/'), + int2lm_run_src=str(cfg.int2lm_run).rstrip('/'), int2lm_run_dest=int2lm_run_path.rstrip('/'), - cosmo_run_src=cfg.cosmo_run.rstrip('/'), + cosmo_run_src=str(cfg.cosmo_run).rstrip('/'), cosmo_run_dest=cosmo_run_path.rstrip('/'), cosmo_output_src=cosmo_output_src, cosmo_output_dest=cosmo_output_dest_path, - logs_src=cfg.log_finished_dir.rstrip('/'), + logs_src=str(cfg.log_finished_dir).rstrip('/'), logs_dest=logs_path.rstrip('/')) # Wait for Cosmo to finish first From 525c58f842b053edc200b30332b6cf53651aecc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 14 Dec 2023 09:23:27 +0100 Subject: [PATCH 013/245] Update jenkins checkers --- jenkins/scripts/jenkins.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/scripts/jenkins.sh b/jenkins/scripts/jenkins.sh index a68ce0bb..ad74a655 100755 --- a/jenkins/scripts/jenkins.sh +++ b/jenkins/scripts/jenkins.sh @@ -73,7 +73,7 @@ else fi # Test COSMO-GHG -if [[ -f work/cosmo-ghg-test/2015010100_6_12/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then +if [[ -f work/cosmo-ghg-test/2015010106_2015010112/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then echo cosmo-ghg test case already finished - skipping test. else echo running cosmo-ghg test case... @@ -81,7 +81,7 @@ else fi # Test COSMO-GHG (spinup) -if [[ -f work/cosmo-ghg-spinup-test/2015010106_-3_6/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then +if [[ -f work/cosmo-ghg-spinup-test/2015010106_2015010112/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then echo cosmo-ghg test case already finished - skipping test. else echo running cosmo-ghg-spinup test case... @@ -89,7 +89,7 @@ else fi # Test ICON -if [[ -f work/icon-test/2018010100_6_12/checkpoints/finished/icon && "$force_execution" == false ]]; then +if [[ -f work/icon-test/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then echo icon test case already finished - skipping test. else echo running icon test case... @@ -97,7 +97,7 @@ else fi # Test ICON-ART -if [[ -f work/icon-art-oem-test/2018010100_0_24/checkpoints/finished/icon && "$force_execution" == false ]]; then +if [[ -f work/icon-art-oem-test/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then echo icon-art test case already finished - skipping test. else echo running icon-art-oem test case... @@ -105,7 +105,7 @@ else fi # Test ICON-ART-GLOBAL -if [[ -f work/icon-art-global-test/2018010100_0_24/checkpoints/finished/icon && "$force_execution" == false ]]; then +if [[ -f work/icon-art-global-test/2018010106_2018010112/checkpoints/finished/icon && "$force_execution" == false ]]; then echo icon-art-global test case already finished - skipping test. else echo running icon-art-global test case... From b33c49b2b1ea6088bda4587f67a4ec4c0aae103c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 14 Dec 2023 11:32:30 +0100 Subject: [PATCH 014/245] Fix for jenkins --- jenkins/scripts/jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/scripts/jenkins.sh b/jenkins/scripts/jenkins.sh index ad74a655..8eb4d26e 100755 --- a/jenkins/scripts/jenkins.sh +++ b/jenkins/scripts/jenkins.sh @@ -81,7 +81,7 @@ else fi # Test COSMO-GHG (spinup) -if [[ -f work/cosmo-ghg-spinup-test/2015010106_2015010112/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then +if [[ -f work/cosmo-ghg-spinup-test/2015010109_2015010118/checkpoints/finished/post_cosmo && "$force_execution" == false ]]; then echo cosmo-ghg test case already finished - skipping test. else echo running cosmo-ghg-spinup test case... From 09bccfc0c22b97bf6b2f4448f885cc567e09142c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 14 Dec 2023 15:06:07 +0100 Subject: [PATCH 015/245] Fixes for sequential runs --- jobs/icon.py | 24 +++++++++--------------- workflows.yaml | 5 ++++- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 6d1943ff..1c742288 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- import logging -import os -import subprocess from . import tools, prepare_data @@ -33,18 +31,17 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - cfg = prepare_data.set_cfg_variables(cfg) + prepare_data.set_cfg_variables(cfg) - logfile = os.path.join(cfg.log_working_dir, "icon") - logfile_finish = os.path.join(cfg.log_finished_dir, "icon") + logfile = cfg.log_working_dir / "icon" + logfile_finish = cfg.log_finished_dir / "icon" logging.info("Setup the namelist for an ICON run and " "submit the job to the queue") # Copy icon executable execname = 'icon.exe' - tools.copy_file(cfg.icon_binary_file, os.path.join(cfg.icon_work, - execname)) + tools.copy_file(cfg.icon_binary_file, cfg.icon_work / execname) # Symlink the restart file to the last run into the icon/run folder if cfg.lrestart == '.TRUE.': @@ -52,13 +49,9 @@ def main(cfg): # Get name of initial file if hasattr(cfg, 'inicond_filename'): - inidata_filename = os.path.join(cfg.icon_input_icbc, - cfg.inicond_filename) + inidata_filename = cfg.icon_input_icbc / cfg.inicond_filename else: - inidata_filename = os.path.join( - cfg.icon_input_icbc, - cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') + inidata_filename = cfg.icon_input_icbc / cfg.startdate_sim.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '.nc' # Write run script (run_icon.job) template = (cfg.case_path / cfg.icon_runjob_filename).read_text() @@ -66,10 +59,11 @@ def main(cfg): inidata_filename=inidata_filename, logfile=logfile, logfile_finish=logfile_finish) - script = (cfg.icon_work / 'run_icon.job').write_text(script_str) + script = (cfg.icon_work / 'run_icon.job') + script.write_text(script_str) # Submit run script - cfg.submit('icon', script) + job_id = cfg.submit('icon', script) # Anything hapenning after submission only makes sense in sequential mode if not cfg.is_async: diff --git a/workflows.yaml b/workflows.yaml index bcbc56f4..45130424 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -9,13 +9,16 @@ cosmo-art: jobs: ['prepare_data', 'emissions', 'obs_nudging', 'photo_rate', 'int2lm', 'cosmo', 'post_cosmo'] features: ['nesting', 'spinup'] -icon: +icon-async: jobs: ['prepare_data', 'icon'] features: ['restart'] dependencies: icon: current: ['prepare_data'] previous: ['icon'] +icon: + jobs: ['prepare_data', 'icon'] + features: ['restart'] icon-art: jobs: ['prepare_data', 'icon'] features: ['restart'] From 750ad67d6bced7234fba485eb9f38978052b1f3b Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 Dec 2023 14:06:37 +0000 Subject: [PATCH 016/245] GitHub Action: Apply Pep8-formatting --- jobs/icon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/icon.py b/jobs/icon.py index 1c742288..39fb2765 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -51,7 +51,8 @@ def main(cfg): if hasattr(cfg, 'inicond_filename'): inidata_filename = cfg.icon_input_icbc / cfg.inicond_filename else: - inidata_filename = cfg.icon_input_icbc / cfg.startdate_sim.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '.nc' + inidata_filename = cfg.icon_input_icbc / cfg.startdate_sim.strftime( + cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '.nc' # Write run script (run_icon.job) template = (cfg.case_path / cfg.icon_runjob_filename).read_text() From 9f374b3e2b554a5ebf7b9714dcc9cdd72923fa92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 14 Dec 2023 19:11:16 +0100 Subject: [PATCH 017/245] Get result from submit script --- config.py | 4 +++- jobs/icon.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index f35cf142..f4aa38e1 100644 --- a/config.py +++ b/config.py @@ -387,6 +387,8 @@ def submit(self, job_name, script, add_dep=None): cwd=script_path.parent, capture_output=True) job_id = int(result.stdout) + print(f"Submitted batch job {job_id}") + if not job_name in self.job_ids['current']: self.job_ids['current'][job_name] = [job_id] else: @@ -394,7 +396,7 @@ def submit(self, job_name, script, add_dep=None): # If needed internaly in a multi-job task like prepare_data # Can then be passed as add_dep keyword - return job_id + return result, job_id def wait_for_previous(self): """wait for all jobs of the previous stage to be finished diff --git a/jobs/icon.py b/jobs/icon.py index 1c742288..afaf8e39 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import logging +import subprocess from . import tools, prepare_data @@ -51,7 +52,7 @@ def main(cfg): if hasattr(cfg, 'inicond_filename'): inidata_filename = cfg.icon_input_icbc / cfg.inicond_filename else: - inidata_filename = cfg.icon_input_icbc / cfg.startdate_sim.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '.nc' + inidata_filename = cfg.icon_input_icbc / str(cfg.startdate_sim.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '.nc') # Write run script (run_icon.job) template = (cfg.case_path / cfg.icon_runjob_filename).read_text() @@ -63,7 +64,7 @@ def main(cfg): script.write_text(script_str) # Submit run script - job_id = cfg.submit('icon', script) + result, job_id = cfg.submit('icon', script) # Anything hapenning after submission only makes sense in sequential mode if not cfg.is_async: From 1f31f0cc311074993402610f934b91fe37efe480 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 Dec 2023 18:12:29 +0000 Subject: [PATCH 018/245] GitHub Action: Apply Pep8-formatting --- jobs/icon.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 130927cb..441628af 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -52,8 +52,9 @@ def main(cfg): if hasattr(cfg, 'inicond_filename'): inidata_filename = cfg.icon_input_icbc / cfg.inicond_filename else: - inidata_filename = cfg.icon_input_icbc / str(cfg.startdate_sim.strftime( - cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '.nc') + inidata_filename = cfg.icon_input_icbc / str( + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '.nc') # Write run script (run_icon.job) template = (cfg.case_path / cfg.icon_runjob_filename).read_text() From fdc454f2b1128d9756bee6f6791d0ffaaa99ea7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 08:40:11 +0100 Subject: [PATCH 019/245] Fix typo --- cases/icon-art-oem-test/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index c7360ea8..06d678d4 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'icon-art-oem-test' case with ICON -worflow: icon-art-oem +workflow: icon-art-oem constraint: gpu run_on: cpu compute_queue: normal From 011f5f96eb4692206004271ecae8b116b6fa74aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 09:02:32 +0100 Subject: [PATCH 020/245] Move copy_input batch job to run --- cases/icon-test/config.yaml | 3 --- jobs/prepare_data.py | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index bbdbd9be..c4f79d0c 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -40,9 +40,6 @@ input_files: lrtm_filename: ./input/icon/rad/rrtmg_lw.nc map_file_ana: ./input/icon/mapping/map_file.ana -prepare_data: - walltime: '00:10:00' - icon: binary_file: ./ext/icon/bin/icon runjob_filename: icon_runjob.cfg diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 3738d45f..c45dad5c 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -106,27 +106,33 @@ def main(cfg): if cfg.workflow_name.startswith('icon'): logging.info('ICON input data (IC/BC)') - #----------------------------------------------------- # Create directories - #----------------------------------------------------- tools.create_dir(cfg.icon_work, "icon_work") tools.create_dir(cfg.icon_input_icbc, "icon_input_icbc") tools.create_dir(cfg.icon_output, "icon_output") tools.create_dir(cfg.icon_restart_out, "icon_restart_out") + # Set logfile + logfile = cfg.log_working_dir / 'prepare_data' + logfile_finish = cfg.log_finished_dir / 'prepare_data' + + # Copy input files to scratch script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', f'#SBATCH --account={cfg.compute_account}', - f'#SBATCH --time={cfg.prepare_data_walltime}', + f'#SBATCH --time=00:10:00', f'#SBATCH --partition={cfg.compute_queue}', - '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', '' + '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', + f'#SBATCH --output={logfile}', + '#SBATCH --open-mode=append', + f'#SBATCH --chdir={cfg.icon_work}','' ] for target, destination in zip(cfg.input_files.values(), cfg.input_files_scratch.values()): script_lines.append(f'rsync -av {target} {destination}') - with (script := cfg.icon_base / 'copy_input.job').open('w') as f: + with (script := cfg.icon_work / 'copy_input.job').open('w') as f: f.write('\n'.join(script_lines)) cfg.submit('prepare_data', script) @@ -322,9 +328,6 @@ def main(cfg): # Write and submit runscripts #----------------------------------------------------- for runscript in cfg.icontools_runjobs: - logfile = os.path.join(cfg.log_working_dir, 'prepare_data') - logfile_finish = os.path.join(cfg.log_finished_dir, - 'prepare_data') with open(os.path.join(cfg.case_path, runscript)) as input_file: to_write = input_file.read() From 63393537690ca8d700abe9070172af75728b7bb5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Dec 2023 08:02:58 +0000 Subject: [PATCH 021/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index c45dad5c..1c708d4c 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -123,10 +123,9 @@ def main(cfg): f'#SBATCH --account={cfg.compute_account}', f'#SBATCH --time=00:10:00', f'#SBATCH --partition={cfg.compute_queue}', - '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', - f'#SBATCH --output={logfile}', - '#SBATCH --open-mode=append', - f'#SBATCH --chdir={cfg.icon_work}','' + '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', + f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', + f'#SBATCH --chdir={cfg.icon_work}', '' ] for target, destination in zip(cfg.input_files.values(), cfg.input_files_scratch.values()): From a90e700534da6ae666f4f6a46ae225cc652c72ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 10:06:56 +0100 Subject: [PATCH 022/245] Add function to check submitted job --- config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index f4aa38e1..897578be 100644 --- a/config.py +++ b/config.py @@ -1,6 +1,7 @@ import subprocess import os import yaml +import logging from jobs import tools from pathlib import Path @@ -387,7 +388,6 @@ def submit(self, job_name, script, add_dep=None): cwd=script_path.parent, capture_output=True) job_id = int(result.stdout) - print(f"Submitted batch job {job_id}") if not job_name in self.job_ids['current']: self.job_ids['current'][job_name] = [job_id] @@ -398,6 +398,12 @@ def submit(self, job_name, script, add_dep=None): # Can then be passed as add_dep keyword return result, job_id + def check_submitted_job(self, script, result): + exitcode = result.returncode + if exitcode != 0: + raise RuntimeError(f"sbatch returned exitcode {exitcode}") + logging.info(f"{script} successfully executed.") + def wait_for_previous(self): """wait for all jobs of the previous stage to be finished From e3962f798ed0132caa0dd59d6be56b32d5a1f2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 10:07:19 +0100 Subject: [PATCH 023/245] Use new interface to submit icontools jobs --- jobs/prepare_data.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index c45dad5c..00cb2f19 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -327,12 +327,14 @@ def main(cfg): #----------------------------------------------------- # Write and submit runscripts #----------------------------------------------------- + last_runscript = None for runscript in cfg.icontools_runjobs: + print(runscript) with open(os.path.join(cfg.case_path, runscript)) as input_file: to_write = input_file.read() - output_run = os.path.join(cfg.icon_work, "%s.job" % runscript) - with open(output_run, "w") as outf: + runscript_path = cfg.icon_work / f"{runscript}.job" + with open(runscript_path, "w") as outf: outf.write( to_write.format(cfg=cfg, meteo=cfg.meteo, @@ -342,15 +344,11 @@ def main(cfg): datafile_list_rest=datafile_list_rest, datafile_list_chem=datafile_list_chem)) logging.info(f" Starting icontools runscript {runscript}.") - result = subprocess.run([ - "sbatch", "--wait", - os.path.join(cfg.icon_work, "%s.job" % runscript) - ]) - exitcode = result.returncode - if exitcode != 0: - raise RuntimeError( - "sbatch returned exitcode {}".format(exitcode)) - logging.info(f"{runscript} successfully executed.") + if last_runscript: + result, last_runscript = cfg.submit('prepare_data', runscript_path, add_dep=last_runscript) + else: + result, last_runscript = cfg.submit('prepare_data', runscript_path) + cfg.check_submitted_job(runscript_path, result) #----------------------------------------------------- # Add GEOSP to all meteo files From 83f74e497536e86fdfada039c1ed73861b14a004 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Dec 2023 09:07:53 +0000 Subject: [PATCH 024/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index fbf96203..c1f3e9a5 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -344,9 +344,12 @@ def main(cfg): datafile_list_chem=datafile_list_chem)) logging.info(f" Starting icontools runscript {runscript}.") if last_runscript: - result, last_runscript = cfg.submit('prepare_data', runscript_path, add_dep=last_runscript) + result, last_runscript = cfg.submit('prepare_data', + runscript_path, + add_dep=last_runscript) else: - result, last_runscript = cfg.submit('prepare_data', runscript_path) + result, last_runscript = cfg.submit( + 'prepare_data', runscript_path) cfg.check_submitted_job(runscript_path, result) #----------------------------------------------------- From 4076b17bb27095122264c262a3d3fad45d53846c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 10:08:19 +0100 Subject: [PATCH 025/245] Remove print --- jobs/prepare_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index fbf96203..743eafd0 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -328,7 +328,6 @@ def main(cfg): #----------------------------------------------------- last_runscript = None for runscript in cfg.icontools_runjobs: - print(runscript) with open(os.path.join(cfg.case_path, runscript)) as input_file: to_write = input_file.read() From 4c76f99ec828bd5f65829aea694e23d002ae3fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 14:27:51 +0100 Subject: [PATCH 026/245] Fix waiting function and add wait parameter for others --- config.py | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/config.py b/config.py index 897578be..af32756a 100644 --- a/config.py +++ b/config.py @@ -125,11 +125,6 @@ def set_account(self): This method determines the compute account to be used based on the user's name and system configuration. - Returns - ------- - Config - The same `Config` instance with the `compute_account` attribute set. - Notes ----- - If the user name is 'jenkins', the compute account is set to 'g110' for @@ -291,11 +286,6 @@ def convert_paths_to_absolute(self, dct=None): Recursively convert all strings starting with './' in the instance attributes to absolute paths. - - Returns - ------- - Config - The same `Config` instance with relative file paths converted to absolute paths. """ if dct is None: self.convert_paths_to_absolute(dct=vars(self)) @@ -313,11 +303,6 @@ def create_vars_from_dicts(self, dct=None, key=None): and checks for dictionary values. For each dictionary encountered, it creates new instance attributes by concatenating the original attribute name and the dictionary key, and assigns the corresponding values. - - Returns - ------- - Config - The same `Config` instance with new attributes created from dictionary entries. """ if dct is None: @@ -359,10 +344,13 @@ def get_dep_ids(self, job_name, add_dep=None): dep_id_list.extend(dep_id) return dep_id_list - def get_dep_cmd(self, job_name, add_dep=None): + def get_dep_cmd(self, job_name, add_dep=None, wait=False): """Generate the part of the sbatch command that sepcifies dependencies for job_name.""" - if self.is_async: + if wait: + # forced wait + return '--wait' + elif self.is_async: # async case if dep_ids := self.get_dep_ids(job_name, add_dep=add_dep): dep_str = ':'.join(map(str, dep_ids)) @@ -375,12 +363,12 @@ def get_dep_cmd(self, job_name, add_dep=None): # sequential case return '--wait' - def submit(self, job_name, script, add_dep=None): + def submit(self, job_name, script, add_dep=None, wait=False): """Submit job with dependencies""" script_path = Path(script) sbatch_cmd = ['sbatch', '--parsable'] - if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): + if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep, wait=wait): sbatch_cmd.append(dep_cmd) sbatch_cmd.append(script_path.name) @@ -407,7 +395,8 @@ def check_submitted_job(self, script, result): def wait_for_previous(self): """wait for all jobs of the previous stage to be finished - Do this by submitting a fake job depending on all jobs from the 'previous' stage. + Do this by submitting a fake job depending on all jobs from the + 'previous' stage. """ dep_ids = [] @@ -415,11 +404,20 @@ def wait_for_previous(self): dep_ids.extend(ids) if dep_ids: job_file = 'submit.wait.slurm' + dep_str = ':'.join(map(str, dep_ids)) + script_lines = [ + '#!/usr/bin/env bash', + f'#SBATCH --job-name="wait"', + f'#SBATCH --nodes=1', + f'#SBATCH --account={self.compute_account}', + f'#SBATCH --partition={self.compute_queue}', + f'#SBATCH --constraint={self.constraint}', + f'#SBATCH --dependency=afterok:{dep_str}', + '', + '# Do nothing', + 'exit 0' + ] with open(job_file, mode='w') as wait_job: - wait_job.write("""#!/bin/bash\n#Do nothing\nexit 0""") + wait_job.write('\n'.join(script_lines)) - subprocess.run([ - 'sbatch', '-W', '--nodes=1', '--job-name=wait', - f'--account={self.compute_account}', job_file - ], - check=True) + subprocess.run(['sbatch', '--wait', job_file], check=True) From b7c0131045355ab6ee82b4c93d26ddf1e0a31243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 14:28:49 +0100 Subject: [PATCH 027/245] Submit icontools runscripts sequentially --- jobs/prepare_data.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 019188eb..14e09bd6 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -123,7 +123,7 @@ def main(cfg): f'#SBATCH --account={cfg.compute_account}', f'#SBATCH --time=00:10:00', f'#SBATCH --partition={cfg.compute_queue}', - '#SBATCH --constraint=gpu', '#SBATCH --nodes=1', + f'#SBATCH --constraint={cfg.constraint}', '#SBATCH --nodes=1', f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', f'#SBATCH --chdir={cfg.icon_work}', '' ] @@ -134,7 +134,7 @@ def main(cfg): with (script := cfg.icon_work / 'copy_input.job').open('w') as f: f.write('\n'.join(script_lines)) - cfg.submit('prepare_data', script) + cfg.submit('prepare_data', script, wait=True) if cfg.workflow_name == 'icon-art-global': async_error(cfg, part='global ICON-ART') @@ -298,7 +298,6 @@ def main(cfg): process.communicate() else: # non-global ICON-ART - async_error(cfg, part='non-global ICON-ART') #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) #----------------------------------------------------- @@ -326,7 +325,6 @@ def main(cfg): #----------------------------------------------------- # Write and submit runscripts #----------------------------------------------------- - last_runscript = None for runscript in cfg.icontools_runjobs: with open(os.path.join(cfg.case_path, runscript)) as input_file: @@ -341,15 +339,10 @@ def main(cfg): datafile_list=datafile_list, datafile_list_rest=datafile_list_rest, datafile_list_chem=datafile_list_chem)) + + # Submitting icontools runscripts sequentially logging.info(f" Starting icontools runscript {runscript}.") - if last_runscript: - result, last_runscript = cfg.submit('prepare_data', - runscript_path, - add_dep=last_runscript) - else: - result, last_runscript = cfg.submit( - 'prepare_data', runscript_path) - cfg.check_submitted_job(runscript_path, result) + cfg.submit(runscript, runscript_path, wait=True) #----------------------------------------------------- # Add GEOSP to all meteo files From 77512cd46b4d81f7f861241e468631f5536e58d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 14:29:33 +0100 Subject: [PATCH 028/245] Refer to chunks, some formatting --- run_chain.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/run_chain.py b/run_chain.py index 7eb6c96a..efdfac5b 100755 --- a/run_chain.py +++ b/run_chain.py @@ -85,7 +85,7 @@ def parse_arguments(): def run_chain(cfg, force, resume): - """Run the processing chain, managing job execution and logging. + """Run a chunk of the processing chain, managing job execution and logging. This function sets up and manages the execution of a processing chain, handling job execution, logging, and various configuration settings. @@ -199,13 +199,14 @@ def run_chain(cfg, force, resume): cfg.convert_gas = True if cfg.is_async: - # Submit current chunck + # Submit current chunk # - [ ] This bypasses all the logfile moving/checking # - [ ] Still needs a mechanism for resume for job in cfg.jobs: + print(f' └── Process "{job}" for chunk "{cfg.job_id}"') getattr(jobs, job).main(cfg) - # wait for previsouy chunk to be done + # wait for previous chunk to be done cfg.wait_for_previous() # cycle cfg.job_ids['previous'] = cfg.job_ids['current'] @@ -219,14 +220,14 @@ def run_chain(cfg, force, resume): if not force: while True: if (log_finished_dir / job).exists(): - print(f"Skip {job} for chain {cfg.job_id}") + print(f"Skip {job} for chunk {cfg.job_id}") skip = True break elif resume: resume = False break else: - print(f"Wait for {job} of chain {cfg.job_id}") + print(f"Wait for {job} of chunk {cfg.job_id}") sys.stdout.flush() for _ in range(3000): time.sleep(0.1) @@ -235,7 +236,7 @@ def run_chain(cfg, force, resume): (log_finished_dir / job).unlink(missing_ok=True) if not skip: - print('Process "%s" for chain "%s"' % (job, cfg.job_id)) + print(f' └── Process "{job}" for chunk "{cfg.job_id}"') sys.stdout.flush() try_count = 1 + (cfg.ntry - 1) * (job == 'cosmo') @@ -310,7 +311,7 @@ def restart_runs(cfg, force, resume): # Set restart variable (only takes effect for ICON) cfg.lrestart = '.FALSE.' if startdate_sim == cfg.startdate else '.TRUE.' - print(f"Starting run with startdate {startdate_sim}") + print(f"└── Starting chunk with startdate {startdate_sim}") cfg.startdate_sim = startdate_sim cfg.enddate_sim = enddate_sim From 7721be5675b0f06ddf411d78fc2619075c99392e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 15 Dec 2023 14:29:46 +0100 Subject: [PATCH 029/245] Add icon async case --- cases/icon-async-test/config.yaml | 52 +++ cases/icon-async-test/icon_runjob.cfg | 345 ++++++++++++++++++ .../icontools_remap_00_lbc_runjob.cfg | 154 ++++++++ .../icontools_remap_ic_runjob.cfg | 245 +++++++++++++ .../icontools_remap_lbc_rest_runjob.cfg | 128 +++++++ 5 files changed, 924 insertions(+) create mode 100644 cases/icon-async-test/config.yaml create mode 100755 cases/icon-async-test/icon_runjob.cfg create mode 100755 cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg create mode 100755 cases/icon-async-test/icontools_remap_ic_runjob.cfg create mode 100755 cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg diff --git a/cases/icon-async-test/config.yaml b/cases/icon-async-test/config.yaml new file mode 100644 index 00000000..1000bebf --- /dev/null +++ b/cases/icon-async-test/config.yaml @@ -0,0 +1,52 @@ +# Configuration file for the 'icon-async-test' case with ICON + +workflow: icon-async +constraint: gpu +run_on: cpu +compute_queue: normal +ntasks_per_node: 12 +restart_step: PT6H +startdate: 2018-01-01T00:00:00Z +enddate: 2018-01-01T12:00:00Z + +eccodes_dir: ./input/eccodes_definitions +iconremap_bin: iconremap +iconsub_bin: iconsub +latbc_filename: ifs_201801_lbc.nc +inidata_filename: ifs_init_2018010100.nc +output_filename: NWP_LAM +filename_format: _DOM_ +lateral_boundary_grid_order: lateral_boundary + +meteo: + dir: ./input/meteo + prefix: ifs_ + nameformat: '%Y%m%d%H' + suffix: .grb + inc: 3 + +icontools_runjobs: + - icontools_remap_ic_runjob.cfg + - icontools_remap_00_lbc_runjob.cfg + - icontools_remap_lbc_rest_runjob.cfg + +input_files: + radiation_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.parent.nc + dynamics_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.nc + map_file_latbc: ./input/icon/grid/map_file.latbc + lateral_boundary_grid: ./input/icon/grid/lateral_boundary.grid.nc + extpar_filename: ./input/icon/grid/external_parameter_icon_VERIFY_DOM_DOM01_tiles.nc + cldopt_filename: ./input/icon/rad/rrtm_cldopt.nc + lrtm_filename: ./input/icon/rad/rrtmg_lw.nc + map_file_ana: ./input/icon/mapping/map_file.ana + +icon: + binary_file: ./ext/icon/bin/icon + runjob_filename: icon_runjob.cfg + compute_queue: normal + walltime: '00:10:00' + np_tot: 8 + np_io: 1 + np_restart: 1 + np_prefetch: 1 + diff --git a/cases/icon-async-test/icon_runjob.cfg b/cases/icon-async-test/icon_runjob.cfg new file mode 100755 index 00000000..09ff2c3a --- /dev/null +++ b/cases/icon-async-test/icon_runjob.cfg @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --time={cfg.icon_walltime} +#SBATCH --nodes={cfg.icon_np_tot} +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --hint=nomultithread +#SBATCH --output={logfile} +#SBATCH --open-mode=append +#SBATCH --chdir={cfg.icon_work} + +# OpenMP environment variables +# ---------------------------- +export OMP_NUM_THREADS=1 +export ICON_THREADS=1 +export OMP_SCHEDULE=static,12 +export OMP_DYNAMIC="false" +export OMP_STACKSIZE=200M +# +# MPI variables +# ------------- +no_of_nodes={cfg.icon_np_tot} +mpi_procs_pernode={cfg.ntasks_per_node} +((mpi_total_procs=no_of_nodes * mpi_procs_pernode)) +# +# blocking length +# --------------- +nproma=16 + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +# ---------------------------------------------------------------------------- +# create ICON master namelist +# ---------------------------------------------------------------------------- + +cat > icon_master.namelist << EOF +! master_nml: ---------------------------------------------------------------- +&master_nml + lrestart = {cfg.lrestart} ! .TRUE.=current experiment is resumed + read_restart_namelists = .true. +/ + +! master_time_control_nml: --------------------------------------------------- +&master_time_control_nml + calendar = 'proleptic gregorian' + restartTimeIntval = '{cfg.restart_step}' + checkpointTimeIntval = '{cfg.restart_step}' + experimentStartDate = '{cfg.ini_datetime_string}' + experimentStopDate = '{cfg.end_datetime_string}' +/ + +! master_model_nml: repeated for each model ---------------------------------- +&master_model_nml + model_type = 1 ! identifies which component to run (atmosphere,ocean,...) + model_name = "ATMO" ! character string for naming this component. + model_namelist_filename = "NAMELIST_{cfg.casename}" ! file name containing the model namelists + model_min_rank = 1 ! start MPI rank for this model + model_max_rank = 65536 ! end MPI rank for this model + model_inc_rank = 1 ! stride of MPI ranks +/ +EOF + +# ---------------------------------------------------------------------- +# model namelists +# ---------------------------------------------------------------------- + +cat > NAMELIST_{cfg.casename} << EOF +! parallel_nml: MPI parallelization ------------------------------------------- +¶llel_nml + nproma = 128 ! loop chunk length + p_test_run = .FALSE. ! .TRUE. means verification run for MPI parallelization + num_io_procs = {cfg.icon_np_io} ! number of I/O processors + num_restart_procs = {cfg.icon_np_restart} ! number of restart processors + num_prefetch_proc = {cfg.icon_np_prefetch} ! number of processors for LBC prefetching + iorder_sendrecv = 3 ! sequence of MPI send/receive calls +/ + + +! run_nml: general switches --------------------------------------------------- +&run_nml + ltestcase = .FALSE. ! real case run + num_lev = 60 ! number of full levels (atm.) for each domain + lvert_nest = .FALSE. ! no vertical nesting + dtime = 60. ! timestep in seconds + ldynamics = .TRUE. ! compute adiabatic dynamic tendencies + ltransport = .TRUE. ! compute large-scale tracer transport + ntracer = 0 ! number of advected tracers + iforcing = 3 ! forcing of dynamics and transport by parameterized processes + msg_level = 13 ! detailed report during integration + ltimer = .TRUE. ! timer for monitoring the runtime of specific routines + timers_level = 10 ! performance timer granularity + check_uuid_gracefully = .TRUE. ! give only warnings for non-matching uuids + output = "nml" ! main switch for enabling/disabling components of the model output + lart = .FALSE. ! main switch for ART + debug_check_level = 10 + restart_filename = "{cfg.icon_restart_out}/{cfg.output_filename}_.nc" + activate_sync_timers = .TRUE. +/ + +! diffusion_nml: horizontal (numerical) diffusion ---------------------------- +&diffusion_nml + lhdiff_vn = .TRUE. ! diffusion on the horizontal wind field + lhdiff_temp = .TRUE. ! diffusion on the temperature field + lhdiff_w = .TRUE. ! diffusion on the vertical wind field + hdiff_order = 5 ! order of nabla operator for diffusion + itype_vn_diffu = 1 ! reconstruction method used for Smagorinsky diffusion + itype_t_diffu = 2 ! discretization of temperature diffusion + hdiff_efdt_ratio = 24.0 ! ratio of e-folding time to time step + hdiff_smag_fac = 0.025 ! scaling factor for Smagorinsky diffusion +/ + +! dynamics_nml: dynamical core ----------------------------------------------- +&dynamics_nml + iequations = 3 ! type of equations and prognostic variables + idiv_method = 1 ! method for divergence computation + divavg_cntrwgt = 0.50 ! weight of central cell for divergence averaging + lcoriolis = .TRUE. ! Coriolis force +/ + +! extpar_nml: external data -------------------------------------------------- +&extpar_nml + itopo = 1 ! topography (0:analytical) + extpar_filename = '{cfg.input_files_scratch_extpar_filename}' ! filename of external parameter input file + n_iter_smooth_topo = 1,1 ! iterations of topography smoother + heightdiff_threshold = 3000. ! height difference between neighb. grid points + hgtdiff_max_smooth_topo = 750.,750. ! see Namelist doc + heightdiff_threshold = 2250.,1500. +/ + +! initicon_nml: specify read-in of initial state ------------------------------ +&initicon_nml + init_mode = 2 ! 7: start from DWD fg with subsequent vertical remapping + lread_ana = .FALSE. ! no analysis data will be read + ifs2icon_filename = "{inidata_filename}" ! initial data filename + ana_varnames_map_file = "{cfg.input_files_scratch_map_file_ana}" ! dictionary mapping internal names onto GRIB2 shortNames + ltile_coldstart = .TRUE. ! coldstart for surface tiles + ltile_init = .FALSE. ! set it to .TRUE. if FG data originate from run without tiles +/ + +! grid_nml: horizontal grid -------------------------------------------------- +&grid_nml + dynamics_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" ! array of the grid filenames for the dycore + radiation_grid_filename = "{cfg.input_files_scratch_radiation_grid_filename}" ! array of the grid filenames for the radiation model + dynamics_parent_grid_id = 0 ! array of the indexes of the parent grid filenames + lredgrid_phys = .TRUE. ! .true.=radiation is calculated on a reduced grid + lfeedback = .TRUE. ! specifies if feedback to parent grid is performed + l_limited_area = .TRUE. ! .TRUE. performs limited area run + ifeedback_type = 2 ! feedback type (incremental/relaxation-based) + start_time = 0. ! Time when a nested domain starts to be active [s] +/ + +! gridref_nml: grid refinement settings -------------------------------------- +&gridref_nml + denom_diffu_v = 150. ! denominator for lateral boundary diffusion of velocity +/ + +! interpol_nml: settings for internal interpolation methods ------------------ +&interpol_nml + nudge_zone_width = 8 ! width of lateral boundary nudging zone + support_baryctr_intp = .FALSE. ! barycentric interpolation support for output + nudge_max_coeff = 0.07 + nudge_efold_width = 2.0 +/ + +! io_nml: general switches for model I/O ------------------------------------- +&io_nml + itype_pres_msl = 5 ! method for computation of mean sea level pressure + itype_rh = 1 ! method for computation of relative humidity + lmask_boundary = .TRUE. ! mask out interpolation zone in output + restart_file_type = 5 +/ + +! limarea_nml: settings for limited area mode --------------------------------- +&limarea_nml + itype_latbc = 1 ! 1: time-dependent lateral boundary conditions + dtime_latbc = 10800 ! time difference between 2 consecutive boundary data + nlev_latbc = 90 ! Number of vertical levels in boundary data + latbc_boundary_grid = "{cfg.input_files_scratch_lateral_boundary_grid}" ! Grid file defining the lateral boundary + latbc_path = "{cfg.icon_input_icbc}" ! Absolute path to boundary data + latbc_varnames_map_file = "{cfg.input_files_scratch_map_file_latbc}" + latbc_filename = "{cfg.latbc_filename}" ! boundary data input filename + init_latbc_from_fg = .FALSE. ! .TRUE.: take lbc for initial time from first guess +/ + +! lnd_nml: land scheme switches ----------------------------------------------- +&lnd_nml + ntiles = 1 ! number of tiles + nlev_snow = 3 ! number of snow layers + lmulti_snow = .FALSE. ! .TRUE. for use of multi-layer snow model + idiag_snowfrac = 20 ! type of snow-fraction diagnosis + lsnowtile = .TRUE. ! .TRUE.=consider snow-covered and snow-free separately + itype_root = 2 ! root density distribution + itype_heatcond = 3 ! type of soil heat conductivity + itype_lndtbl = 4 ! table for associating surface parameters + itype_evsl = 4 ! type of bare soil evaporation + itype_root = 2 ! root density distribution + cwimax_ml = 5.e-4 ! scaling parameter for max. interception storage + c_soil = 1.75 ! surface area density of the evaporative soil surface + c_soil_urb = 0.5 ! same for urban areas + lseaice = .TRUE. ! .TRUE. for use of sea-ice model + llake = .TRUE. ! .TRUE. for use of lake model +/ + +! nonhydrostatic_nml: nonhydrostatic model ----------------------------------- +&nonhydrostatic_nml + iadv_rhotheta = 2 ! advection method for rho and rhotheta + ivctype = 2 ! type of vertical coordinate + itime_scheme = 4 ! time integration scheme + ndyn_substeps = 5 ! number of dynamics steps per fast-physics step + exner_expol = 0.333 ! temporal extrapolation of Exner function + vwind_offctr = 0.2 ! off-centering in vertical wind solver + damp_height = 12500.0 ! height at which Rayleigh damping of vertical wind starts + rayleigh_coeff = 1.5 ! Rayleigh damping coefficient + divdamp_order = 24 ! order of divergence damping + divdamp_type = 3 ! type of divergence damping + divdamp_fac = 0.004 ! scaling factor for divergence damping + l_open_ubc = .FALSE. ! .TRUE.=use open upper boundary condition + igradp_method = 3 ! discretization of horizontal pressure gradient + l_zdiffu_t = .TRUE. ! specifies computation of Smagorinsky temperature diffusion + thslp_zdiffu = 0.02 ! slope threshold (temperature diffusion) + thhgtd_zdiffu = 125.0 ! threshold of height difference (temperature diffusion) + htop_moist_proc = 22500.0 ! max. height for moist physics + hbot_qvsubstep = 22500.0 ! height above which QV is advected with substepping scheme +/ + +! nwp_phy_nml: switches for the physics schemes ------------------------------ +&nwp_phy_nml + inwp_gscp = 2 ! cloud microphysics and precipitation + inwp_convection = 1 ! convection + lshallowconv_only = .FALSE. ! only shallow convection + inwp_radiation = 1 ! radiation + inwp_cldcover = 1 ! cloud cover scheme for radiation + inwp_turb = 1 ! vertical diffusion and transfer + inwp_satad = 1 ! saturation adjustment + inwp_sso = 1 ! subgrid scale orographic drag + inwp_gwd = 0 ! non-orographic gravity wave drag + inwp_surface = 1 ! surface scheme + latm_above_top = .TRUE. ! take into account atmosphere above model top for radiation computation + ldetrain_conv_prec = .TRUE. + efdt_min_raylfric = 7200. ! minimum e-folding time of Rayleigh friction + itype_z0 = 2 ! type of roughness length data + icapdcycl = 3 ! apply CAPE modification to improve diurnalcycle over tropical land + icpl_aero_conv = 1 ! coupling between autoconversion and Tegen aerosol climatology + icpl_aero_gscp = 1 ! coupling between autoconversion and Tegen aerosol climatology + lrtm_filename = '{cfg.input_files_scratch_lrtm_filename}' ! longwave absorption coefficients for RRTM_LW + cldopt_filename = '{cfg.input_files_scratch_cldopt_filename}' ! RRTM cloud optical properties + dt_rad = 720. ! time step for radiation in s + dt_conv = 120.,90.,90. ! time step for convection in s (domain specific) + dt_sso = 120.,360.,360. ! time step for SSO parameterization + dt_gwd = 360.,360.,360. ! time step for gravity wave drag parameterization +/ + +! nwp_tuning_nml: additional tuning parameters ---------------------------------- +&nwp_tuning_nml + itune_albedo = 1 ! reduced albedo (w.r.t. MODIS data) over Sahara + tune_gkwake = 1.8 + tune_gkdrag = 0.01 + tune_minsnowfrac = 0.3 +/ + +! output_nml: specifies an output stream -------------------------------------- +&output_nml + filetype = 4 ! output format: 2=GRIB2, 4=NETCDFv2 + dom = 1 ! write domain 1 only + output_bounds = 0., 10000000., 3600. ! start, end, increment + steps_per_file = 1 ! number of steps per file + mode = 1 ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis) + include_last = .TRUE. + output_filename = '{cfg.output_filename}' + filename_format = '{cfg.icon_output}/_DOM_' ! file name base + steps_per_file_inclfirst = .FALSE. + output_grid = .TRUE. + remap = 1 ! 1: remap to lat-lon grid + !north_pole = -170.,40. ! definition of north_pole for rotated lat-lon grid + reg_lon_def = -16.0,0.13,36.0 ! + reg_lat_def = 32.0,0.12,74.0 ! + ml_varlist = 'group:PBL_VARS', + 'group:ATMO_ML_VARS', + 'group:precip_vars', + 'group:land_vars', + 'group:nh_prog_vars', + 'z_mc', 'z_ifc', +/ + +! radiation_nml: radiation scheme --------------------------------------------- +&radiation_nml + irad_o3 = 7 ! ozone climatology + irad_aero = 6 ! aerosols + albedo_type = 2 ! type of surface albedo + vmr_co2 = 390.e-06 + vmr_ch4 = 1800.e-09 + vmr_n2o = 322.0e-09 + vmr_o2 = 0.20946 + vmr_cfc11 = 240.e-12 + vmr_cfc12 = 532.e-12 +/ + +! sleve_nml: vertical level specification ------------------------------------- +&sleve_nml + min_lay_thckn = 20.0 ! layer thickness of lowermost layer + top_height = 23000.0 ! height of model top + stretch_fac = 0.65 ! stretching factor to vary distribution of model levels + decay_scale_1 = 4000.0 ! decay scale of large-scale topography component + decay_scale_2 = 2500.0 ! decay scale of small-scale topography component + decay_exp = 1.2 ! exponent of decay function + flat_height = 16000.0 ! height above which the coordinate surfaces are flat +/ + +! transport_nml: tracer transport --------------------------------------------- +&transport_nml + npassive_tracer = 0 ! number of additional passive tracers + ivadv_tracer = 3, 3, 3, 3, 3, 3 ! tracer specific method to compute vertical advection + itype_hlimit = 3, 4, 4, 4, 4, 4 ! type of limiter for horizontal transport + ihadv_tracer = 52, 2, 2, 2, 2, 22 ! tracer specific method to compute horizontal advection + llsq_svd = .TRUE. ! use SV decomposition for least squares design matrix +/ + +! turbdiff_nml: turbulent diffusion ------------------------------------------- +&turbdiff_nml + tkhmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient + tkmmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient + pat_len = 750.0 ! effective length scale of thermal surface patterns + c_diff = 0.2 ! length scale factor for vertical diffusion of TKE + rat_sea = 7.5 ! controls laminar resistance for sea surface + rlam_heat = 1.5 + ltkesso = .TRUE. ! consider TKE-production by sub-grid SSO wakes + frcsmot = 0.2 ! these 2 switches together apply vertical smoothing of the TKE source terms + imode_frcsmot = 2 ! in the tropics (only), which reduces the moist bias in the tropical lower troposphere + itype_sher = 3 ! type of shear forcing used in turbulence + ltkeshs = .TRUE. ! include correction term for coarse grids in hor. shear production term + a_hshr = 2.0 ! length scale factor for separated horizontal shear mode + icldm_turb = 1 ! mode of cloud water representation in turbulence + ldiff_qi = .TRUE. +/ + +EOF + +# ---------------------------------------------------------------------- +# run the model! +# ---------------------------------------------------------------------- + srun ./icon.exe diff --git a/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg new file mode 100755 index 00000000..a93ac550 --- /dev/null +++ b/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +#SBATCH --job-name="iconsub_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --chdir={cfg.icon_work} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --nodes=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --output={logfile} +#SBATCH --open-mode=append + +ulimit -s unlimited + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh +spack load icontools + +#----------------------------------------------------------------------------- +# PART I: Create auxiliary grid file which contains only the cells of the +# boundary zone. +#----------------------------------------------------------------------------- +cat > NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} << EOF_1 +&iconsub_nml + grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}', + output_type = 4, + lwrite_grid = .TRUE., +/ +&subarea_nml + ORDER = "{cfg.lateral_boundary_grid_order}", + grf_info_file = '{cfg.input_files_scratch_dynamics_grid_filename}', + min_refin_c_ctrl = 1 + max_refin_c_ctrl = 14 +/ +EOF_1 + +srun -n 1 {cfg.iconsub_bin} \ + --nml NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} 2>&1 + +#----------------------------------------------------------------------------- +# PART II: Extract boundary data +#----------------------------------------------------------------------------- +rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* + +set +x +cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF +! +&input_field_nml ! temperature + inputname = "T" + outputname = "T" + code = 130 + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "U" + outputname = "U" + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "V" + outputname = "V" + intp_method = 3 +/ +&input_field_nml ! vertical velocity + inputname = "OMEGA" + outputname = "W" + code = 135 + intp_method = 3 +/ +&input_field_nml ! surface pressure + inputname = "LNSP" + outputname = "LNPS" + code = 152 + intp_method = 3 +/ +&input_field_nml ! geopotential + inputname = "Z" + outputname = "GEOSP" + code = 129 + intp_method = 3 +/ +&input_field_nml ! specific humidity + inputname = "QV" + outputname = "QV" + code = 133 + intp_method = 3 +/ +&input_field_nml ! cloud liquid water content + inputname = "CLWC" + outputname = "QC" + code = 246 + intp_method = 3 +/ +&input_field_nml ! cloud ice water content + inputname = "CIWC" + outputname = "QI" + code = 247 + intp_method = 3 +/ +&input_field_nml ! rain water content + inputname = "CRWC" + outputname = "QR" + code = 75 + intp_method = 3 +/ +&input_field_nml ! snow water content + inputname = "CSWC" + outputname = "QS" + code = 76 + intp_method = 3 +/ +EOF + +#----------------------------------------------------------------------------- +# loop over file list: + +echo "DATAFILELIST is {datafile_list}" +for datafilename in {datafile_list} ; do + datafile="${{datafilename##*/}}" # get filename without path + outdatafile=${{datafile%.*}} # get filename without suffix + cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C +&remap_nml + in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' + in_filename = '{cfg.meteo_dir}/${{datafile}}' + in_type = 1 + out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' + out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' + out_type = 2 + out_filetype = 4 + l_have3dbuffer = .false. +! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" +/ +EOF_2C + + srun -n 1 {cfg.iconremap_bin} -q \ + --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ + --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 + +done + +#----------------------------------------------------------------------------- +# clean-up + +rm -f nml.log +rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} + +#----------------------------------------------------------------------------- +exit +#----------------------------------------------------------------------------- + diff --git a/cases/icon-async-test/icontools_remap_ic_runjob.cfg b/cases/icon-async-test/icontools_remap_ic_runjob.cfg new file mode 100755 index 00000000..e31b9eb5 --- /dev/null +++ b/cases/icon-async-test/icontools_remap_ic_runjob.cfg @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +#SBATCH --job-name="iconremap_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --chdir={cfg.icon_work} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --nodes=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --output={logfile} +#SBATCH --open-mode=append + +ulimit -s unlimited + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh +spack load icontools + +#----------------------------------------------------------------------------- +# Remap inital data onto local (limited-area) grid +#----------------------------------------------------------------------------- +cat > NAMELIST_ICONREMAP_FIELDS << EOF +! +&input_field_nml ! temperature + inputname = "T" + outputname = "T" + code = 130 + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "U" + outputname = "U" + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "V" + outputname = "V" + intp_method = 3 +/ +&input_field_nml ! vertical velocity + inputname = "OMEGA" + outputname = "W" + code = 135 + intp_method = 3 +/ +&input_field_nml ! surface pressure + inputname = "LNSP" + outputname = "LNPS" + code = 152 + intp_method = 3 +/ +&input_field_nml ! geopotential + inputname = "Z" + outputname = "GEOSP" + code = 129 + intp_method = 3 +/ +&input_field_nml ! geopotential + inputname = "Z" + outputname = "GEOP_SFC" + code = 129 + intp_method = 3 +/ +&input_field_nml ! specific humidity + inputname = "QV" + outputname = "QV" + code = 133 + intp_method = 3 +/ +&input_field_nml ! cloud liquid water content + inputname = "CLWC" + outputname = "QC" + code = 246 + intp_method = 3 +/ +&input_field_nml ! cloud ice water content + inputname = "CIWC" + outputname = "QI" + code = 247 + intp_method = 3 +/ +&input_field_nml ! rain water content + inputname = "CRWC" + outputname = "QR" + code = 75 + intp_method = 3 +/ +&input_field_nml ! snow water content + inputname = "CSWC" + outputname = "QS" + code = 76 + intp_method = 3 +/ +&input_field_nml ! snow temperature + inputname = "TSN" + outputname = "T_SNOW" + code = 238 + intp_method = 3 +/ +&input_field_nml ! water content of snow + inputname = "SD" + outputname = "W_SNOW" + code = 141 + intp_method = 3 +/ +&input_field_nml ! density of snow + inputname = "RSN" + outputname = "RHO_SNOW" + code = 33 + intp_method = 3 +/ +&input_field_nml ! snow albedo + inputname = "ASN" + outputname = "ALB_SNOW" + code = 32 + intp_method = 3 +/ +&input_field_nml ! skin temperature + inputname = "SKT" + outputname = "SKT" + code = 235 + intp_method = 3 +/ +&input_field_nml ! sea surface temperature + inputname = "SST" + outputname = "SST" + code = 34 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 1 + inputname = "STL1" + outputname = "STL1" + code = 139 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 2 + inputname = "STL2" + outputname = "STL2" + code = 170 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 3 + inputname = "STL3" + outputname = "STL3" + code = 183 + intp_method = 3 +/ +&input_field_nml ! soil temperature level 4 + inputname = "STL4" + outputname = "STL4" + code = 236 + intp_method = 3 +/ +&input_field_nml ! sea-ice cover + inputname = "CI" + outputname = "CI" + code = 31 + intp_method = 3 +/ +&input_field_nml ! water cont. of interception storage + inputname = "SRC" + outputname = "W_I" + code = 198 + intp_method = 3 +/ +&input_field_nml ! surface roughness + inputname = "SR" + outputname = "Z0" + code = 173 + intp_method = 3 +/ +&input_field_nml ! Land/sea mask + inputname = "LSM" + outputname = "LSM" + code = 172 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 1 + inputname = "SWVL1" + outputname = "SMIL1" + code = 80 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 2 + inputname = "SWVL2" + outputname = "SMIL2" + code = 81 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 3 + inputname = "SWVL3" + outputname = "SMIL3" + code = 82 + intp_method = 3 +/ +&input_field_nml ! soil moisture index layer 4 + inputname = "SWVL4" + outputname = "SMIL4" + code = 83 + intp_method = 3 +/ +EOF + +#----------------------------------------------------------------------------- +# loop over file list: + +datafilename={cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix} +datafile="${{datafilename##*/}}" # get filename without path +outdatafile=${{datafile%.*}} # get filename without suffix + +# create ICON master namelist +# ------------------------ +# For a complete list see Namelist_overview and Namelist_overview.pdf + +cat > NAMELIST_ICONREMAP << EOF +&remap_nml + in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' + in_filename = '{cfg.meteo_dir}/${{datafile}}' + in_type = 1 + out_grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}' + out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}.nc' + out_type = 2 + out_filetype = 4 + l_have3dbuffer = .false. +/ +EOF + +srun -n 1 {cfg.iconremap_bin} \ + -vvvvv -q --remap_nml NAMELIST_ICONREMAP \ + --input_field_nml NAMELIST_ICONREMAP_FIELDS 2>&1 + + +#----------------------------------------------------------------------------- +# clean-up + +rm -f ncstorage.tmp* +rm -f nml.log NAMELIST_SUB NAMELIST_ICONREMAP NAMELIST_ICONREMAP_FIELDS + +#----------------------------------------------------------------------------- +exit +#----------------------------------------------------------------------------- diff --git a/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg new file mode 100755 index 00000000..b0950efa --- /dev/null +++ b/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +#SBATCH --job-name="iconremap_lbc_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --account={cfg.compute_account} +#SBATCH --chdir={cfg.icon_work} +#SBATCH --partition={cfg.compute_queue} +#SBATCH --constraint={cfg.constraint} +#SBATCH --nodes=1 +#SBATCH --ntasks-per-core=1 +#SBATCH --ntasks-per-node={cfg.ntasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --output={logfile} +#SBATCH --open-mode=append + +ulimit -s unlimited + +set -x + +export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions + +. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh +spack load icontools + +#----------------------------------------------------------------------------- +# Extract boundary data +#----------------------------------------------------------------------------- + +rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* + +cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF +! +&input_field_nml ! temperature + inputname = "T" + outputname = "T" + code = 130 + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. u + inputname = "U" + outputname = "U" + intp_method = 3 +/ +&input_field_nml ! horiz. wind comp. v + inputname = "V" + outputname = "V" + intp_method = 3 +/ +&input_field_nml ! vertical velocity + inputname = "OMEGA" + outputname = "W" + code = 135 + intp_method = 3 +/ +&input_field_nml ! surface pressure + inputname = "LNSP" + outputname = "LNPS" + code = 152 + intp_method = 3 +/ +&input_field_nml ! specific humidity + inputname = "QV" + outputname = "QV" + code = 133 + intp_method = 3 +/ +&input_field_nml ! cloud liquid water content + inputname = "CLWC" + outputname = "QC" + code = 246 + intp_method = 3 +/ +&input_field_nml ! cloud ice water content + inputname = "CIWC" + outputname = "QI" + code = 247 + intp_method = 3 +/ +&input_field_nml ! rain water content + inputname = "CRWC" + outputname = "QR" + code = 75 + intp_method = 3 +/ +&input_field_nml ! snow water content + inputname = "CSWC" + outputname = "QS" + code = 76 + intp_method = 3 +/ +EOF + +#----------------------------------------------------------------------------- +# loop over file list: + +echo "DATAFILELIST is {datafile_list_rest}" +for datafilename in {datafile_list_rest} ; do + datafile="${{datafilename##*/}}" # get filename without path + outdatafile=${{datafile%.*}} # get filename without suffix + cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C +&remap_nml + in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' + in_filename = '{cfg.meteo_dir}/${{datafile}}' + in_type = 1 + out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' + out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' + out_type = 2 + out_filetype = 4 + l_have3dbuffer = .false. +! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" +/ +EOF_2C + + srun -n 1 {cfg.iconremap_bin} -q \ + --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ + --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 + +done + +#----------------------------------------------------------------------------- +# clean-up + +#rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* +rm -f nml.log +rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} + +#----------------------------------------------------------------------------- +exit +#----------------------------------------------------------------------------- + From b670f574def7984e2e41240936b33c4337f27aac Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Dec 2023 13:30:18 +0000 Subject: [PATCH 030/245] GitHub Action: Apply Pep8-formatting --- config.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index af32756a..d516938e 100644 --- a/config.py +++ b/config.py @@ -406,15 +406,12 @@ def wait_for_previous(self): job_file = 'submit.wait.slurm' dep_str = ':'.join(map(str, dep_ids)) script_lines = [ - '#!/usr/bin/env bash', - f'#SBATCH --job-name="wait"', + '#!/usr/bin/env bash', f'#SBATCH --job-name="wait"', f'#SBATCH --nodes=1', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', - f'#SBATCH --dependency=afterok:{dep_str}', - '', - '# Do nothing', + f'#SBATCH --dependency=afterok:{dep_str}', '', '# Do nothing', 'exit 0' ] with open(job_file, mode='w') as wait_job: From 34d4a9b50d2daf4e218b7664fd5a4bb1ce312702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 18 Dec 2023 17:04:23 +0100 Subject: [PATCH 031/245] Move wait script to working dir and delete it after finishing --- config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index d516938e..593b1294 100644 --- a/config.py +++ b/config.py @@ -403,11 +403,13 @@ def wait_for_previous(self): for ids in self.job_ids['previous'].values(): dep_ids.extend(ids) if dep_ids: - job_file = 'submit.wait.slurm' + job_file = self.chain_root / 'submit.wait.slurm' + log_file = self.chain_root / 'wait.log' dep_str = ':'.join(map(str, dep_ids)) script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="wait"', f'#SBATCH --nodes=1', + f'#SBATCH --output={log_file}', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', @@ -418,3 +420,7 @@ def wait_for_previous(self): wait_job.write('\n'.join(script_lines)) subprocess.run(['sbatch', '--wait', job_file], check=True) + + # Remove sbatch script after execution + os.remove(job_file) + os.remove(log_file) \ No newline at end of file From 3896f4ab8ab205a5762b280dbf87ddb26f509443 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 18 Dec 2023 16:04:51 +0000 Subject: [PATCH 032/245] GitHub Action: Apply Pep8-formatting --- config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 593b1294..fbdca3b2 100644 --- a/config.py +++ b/config.py @@ -408,8 +408,7 @@ def wait_for_previous(self): dep_str = ':'.join(map(str, dep_ids)) script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="wait"', - f'#SBATCH --nodes=1', - f'#SBATCH --output={log_file}', + f'#SBATCH --nodes=1', f'#SBATCH --output={log_file}', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', @@ -422,5 +421,5 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) # Remove sbatch script after execution - os.remove(job_file) - os.remove(log_file) \ No newline at end of file + os.remove(job_file) + os.remove(log_file) From d6f9b6c4c67249e971c4fff5990581ea151f7c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 19 Dec 2023 14:30:59 +0100 Subject: [PATCH 033/245] Skip finished jobs and process logfiles --- run_chain.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/run_chain.py b/run_chain.py index efdfac5b..87d9477f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -200,11 +200,19 @@ def run_chain(cfg, force, resume): if cfg.is_async: # Submit current chunk - # - [ ] This bypasses all the logfile moving/checking - # - [ ] Still needs a mechanism for resume for job in cfg.jobs: - print(f' └── Process "{job}" for chunk "{cfg.job_id}"') - getattr(jobs, job).main(cfg) + if (log_finished_dir / job).exists(): + # Skip job if already finished + print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') + skip = True + else: + # Submit job and process logfile + print(f' └── Process "{job}" for chunk "{cfg.job_id}"') + getattr(jobs, job).main(cfg) + logfile = cfg.log_working_dir / job + logfile_finish = cfg.log_finished_dir / job + tools.change_logfile(logfile) + shutil.copy(logfile, logfile_finish) # wait for previous chunk to be done cfg.wait_for_previous() From ae5627fd7307def4a0d5ddd51fed6b0d2111a7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 19 Dec 2023 14:35:59 +0100 Subject: [PATCH 034/245] Add force for async --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 87d9477f..e28f54b6 100755 --- a/run_chain.py +++ b/run_chain.py @@ -201,7 +201,7 @@ def run_chain(cfg, force, resume): if cfg.is_async: # Submit current chunk for job in cfg.jobs: - if (log_finished_dir / job).exists(): + if (log_finished_dir / job).exists() and not force: # Skip job if already finished print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') skip = True From 640677b2eef0b81aca83ad1aa44ab50e9771b3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 19 Dec 2023 14:37:31 +0100 Subject: [PATCH 035/245] Add job submit info --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index fbdca3b2..172c6330 100644 --- a/config.py +++ b/config.py @@ -376,6 +376,7 @@ def submit(self, job_name, script, add_dep=None, wait=False): cwd=script_path.parent, capture_output=True) job_id = int(result.stdout) + print(f' └── Submitted batch job {job_id}') if not job_name in self.job_ids['current']: self.job_ids['current'][job_name] = [job_id] From d4f982cb6dad773ce8a6d67aff66b25094ac635a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 19 Dec 2023 16:48:05 +0100 Subject: [PATCH 036/245] Add option to define workflow in config.yaml --- cases/icon-async-test/config.yaml | 8 +++++++- config.py | 22 ++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cases/icon-async-test/config.yaml b/cases/icon-async-test/config.yaml index 1000bebf..5d96ca6c 100644 --- a/cases/icon-async-test/config.yaml +++ b/cases/icon-async-test/config.yaml @@ -1,6 +1,12 @@ # Configuration file for the 'icon-async-test' case with ICON -workflow: icon-async +workflow: + jobs: ['prepare_data', 'icon'] + features: ['restart'] + dependencies: + icon: + current: ['prepare_data'] + previous: ['icon'] constraint: gpu run_on: cpu compute_queue: normal diff --git a/config.py b/config.py index 172c6330..36dbb680 100644 --- a/config.py +++ b/config.py @@ -56,12 +56,12 @@ def __init__(self, casename): # User-defined attributes from config file self.load_config_file(casename) - # Specific settings based on the node type ('gpu' or 'mc') - self.set_node_info() - # Set workflow and async attributes and initiate job ids dict self.set_workflow() + # Specific settings based on the node type ('gpu' or 'mc') + self.set_node_info() + def load_config_file(self, casename): """Load configuration settings from a YAML file and set them as attributes. @@ -187,10 +187,17 @@ def set_node_info(self): def set_workflow(self): """set workflow and async attr, initiate job ids dict""" + # If a workflow name is specified, load from workflows.yaml + if isinstance(self.workflow, str): + with open('workflows.yaml') as file: + workflows = yaml.safe_load(file) + self.workflow = workflows[self.workflow_name] + # Otherwise, use custom workflow from config.yaml directly + elif isinstance(self.workflow, dict): + self.workflow_name = self.casename + else: + raise InvalidWorkflowType("Invalid workflow type. Must be either a string or a dictionary.") - with open('workflows.yaml') as file: - workflows = yaml.safe_load(file) - self.workflow = workflows[self.workflow_name] self.is_async = 'dependencies' in self.workflow # Initiate empty job ids dictionnary so that it can be filled in later @@ -424,3 +431,6 @@ def wait_for_previous(self): # Remove sbatch script after execution os.remove(job_file) os.remove(log_file) + +class InvalidWorkflowType(Exception): + pass \ No newline at end of file From 18eacad8fc3a97f4f21aea1ff5432ec0b7098572 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 19 Dec 2023 15:48:34 +0000 Subject: [PATCH 037/245] GitHub Action: Apply Pep8-formatting --- config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 36dbb680..3090c80d 100644 --- a/config.py +++ b/config.py @@ -193,10 +193,12 @@ def set_workflow(self): workflows = yaml.safe_load(file) self.workflow = workflows[self.workflow_name] # Otherwise, use custom workflow from config.yaml directly - elif isinstance(self.workflow, dict): + elif isinstance(self.workflow, dict): self.workflow_name = self.casename else: - raise InvalidWorkflowType("Invalid workflow type. Must be either a string or a dictionary.") + raise InvalidWorkflowType( + "Invalid workflow type. Must be either a string or a dictionary." + ) self.is_async = 'dependencies' in self.workflow @@ -432,5 +434,6 @@ def wait_for_previous(self): os.remove(job_file) os.remove(log_file) + class InvalidWorkflowType(Exception): - pass \ No newline at end of file + pass From 5a1ccde5e40a950dcfd4546ea802c61ef2be084f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 20 Dec 2023 11:47:40 +0100 Subject: [PATCH 038/245] run_chain() -> run_chunk() --- run_chain.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/run_chain.py b/run_chain.py index e28f54b6..2fd6d33a 100755 --- a/run_chain.py +++ b/run_chain.py @@ -84,7 +84,7 @@ def parse_arguments(): return args -def run_chain(cfg, force, resume): +def run_chunk(cfg, force, resume): """Run a chunk of the processing chain, managing job execution and logging. This function sets up and manages the execution of a processing chain, handling @@ -291,7 +291,7 @@ def restart_runs(cfg, force, resume): """Start subchains in specified intervals and manage restarts. This function slices the total runtime of the processing chain according to the - `cfg.restart_step_hours` configuration. It calls `run_chain()` for each + `cfg.restart_step_hours` configuration. It calls `run_chunk()` for each specified interval. Parameters @@ -305,10 +305,9 @@ def restart_runs(cfg, force, resume): Notes ----- - - The function iterates over specified intervals, calling `run_chain()` for each. + - The function iterates over specified intervals, calling `run_chunk()` for each. - It manages restart settings and logging for each subchain. """ - # run restarts for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, cfg.restart_step_hours): enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) @@ -324,20 +323,20 @@ def restart_runs(cfg, force, resume): cfg.startdate_sim = startdate_sim cfg.enddate_sim = enddate_sim - run_chain(cfg=cfg, force=force, resume=resume) + run_chunk(cfg=cfg, force=force, resume=resume) def restart_runs_spinup(cfg, force, resume): """Start subchains in specified intervals and manage restarts with spin-up. This function slices the total runtime of the processing chain according to the - `cfg.restart_step_hours` configuration. It calls `run_chain()` for each specified - interval, managing restarts with spin-up. + `cfg.restart_step_hours` configuration. It calls `run_chunk()` for each + specified interval, managing restarts with spin-up. Parameters ---------- cfg : Config - List of names of jobs to execute on every timeslice. + Object holding all user-configuration parameters as attributes. force : bool If True, it will force the execution of jobs regardless of their completion status. resume : bool @@ -345,7 +344,7 @@ def restart_runs_spinup(cfg, force, resume): Notes ----- - - The function iterates over specified intervals, calling `run_chain()` for each. + - The function iterates over specified intervals, calling `run_chunk()` for each. - It manages restart settings and logging for each subchain, including spin-up. """ for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, @@ -380,7 +379,7 @@ def restart_runs_spinup(cfg, force, resume): cfg.startdate_sim = startdate_sim_spinup cfg.enddate_sim = enddate_sim - run_chain(cfg=cfg, force=force, resume=resume) + run_chunk(cfg=cfg, force=force, resume=resume) def main(): @@ -449,7 +448,7 @@ def main(): print("No restarts are used.") cfg.startdate_sim = cfg.startdate cfg.enddate_sim = cfg.enddate - run_chain(cfg=cfg, force=args.force, resume=args.resume) + run_chunk(cfg=cfg, force=args.force, resume=args.resume) print('>>> Finished the processing chain successfully <<<') From 6dd552a22f56a03fe062ae14979c0b6151c9e7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 20 Dec 2023 11:57:57 +0100 Subject: [PATCH 039/245] Use config instead of local variables --- config.py | 2 +- run_chain.py | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/config.py b/config.py index 3090c80d..fc5e36ae 100644 --- a/config.py +++ b/config.py @@ -430,7 +430,7 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) - # Remove sbatch script after execution + # Remove sbatch script and log file after execution os.remove(job_file) os.remove(log_file) diff --git a/run_chain.py b/run_chain.py index 2fd6d33a..1a2e8e3e 100755 --- a/run_chain.py +++ b/run_chain.py @@ -182,15 +182,13 @@ def run_chunk(cfg, force, resume): cfg.meteo.prefix = 'lffd' # Logging - log_working_dir = cfg.chain_root / 'checkpoints' / 'working' - log_finished_dir = cfg.chain_root / 'checkpoints' / 'finished' - cfg.log_working_dir = log_working_dir - cfg.log_finished_dir = log_finished_dir + cfg.log_working_dir = cfg.chain_root / 'checkpoints' / 'working' + cfg.log_finished_dir = cfg.chain_root / 'checkpoints' / 'finished' # Create working directories tools.create_dir(cfg.chain_root, "chain_root") - tools.create_dir(log_working_dir, "log_working") - tools.create_dir(log_finished_dir, "log_finished") + tools.create_dir(cfg.log_working_dir, "log_working") + tools.create_dir(cfg.log_finished_dir, "log_finished") # Number of levels and switch for unit conversion for 'reduce_output' job if not hasattr(cfg, 'output_levels'): @@ -201,7 +199,7 @@ def run_chunk(cfg, force, resume): if cfg.is_async: # Submit current chunk for job in cfg.jobs: - if (log_finished_dir / job).exists() and not force: + if (cfg.log_finished_dir / job).exists() and not force: # Skip job if already finished print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') skip = True @@ -224,10 +222,10 @@ def run_chunk(cfg, force, resume): skip = False # if exists job is currently worked on or has been finished - if (log_working_dir / job).exists(): + if (cfg.log_working_dir / job).exists(): if not force: while True: - if (log_finished_dir / job).exists(): + if (cfg.log_finished_dir / job).exists(): print(f"Skip {job} for chunk {cfg.job_id}") skip = True break @@ -240,8 +238,8 @@ def run_chunk(cfg, force, resume): for _ in range(3000): time.sleep(0.1) else: - (log_working_dir / job).unlink() - (log_finished_dir / job).unlink(missing_ok=True) + (cfg.log_working_dir / job).unlink() + (cfg.log_finished_dir / job).unlink(missing_ok=True) if not skip: print(f' └── Process "{job}" for chunk "{cfg.job_id}"') @@ -269,7 +267,7 @@ def run_chunk(cfg, force, resume): job, cfg.job_id) logging.exception(subject) if cfg.user_mail: - message = tools.prepare_message(log_working_dir / + message = tools.prepare_message(cfg.log_working_dir / job) logging.info('Sending log file to %s' % cfg.user_mail) @@ -277,11 +275,11 @@ def run_chunk(cfg, force, resume): if try_count == 0: raise RuntimeError(subject) - if exitcode != 0 or not (log_finished_dir / job).exists(): + if exitcode != 0 or not (cfg.log_finished_dir / job).exists(): subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( job, cfg.job_id) if cfg.user_mail: - message = tools.prepare_message(log_working_dir / job) + message = tools.prepare_message(cfg.log_working_dir / job) logging.info('Sending log file to %s' % cfg.user_mail) tools.send_mail(cfg.user_mail, subject, message) raise RuntimeError(subject) From 7ec7bc199e5bcb60df16ab41ca77f6a08ea1292b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 20 Dec 2023 13:51:55 +0100 Subject: [PATCH 040/245] Add logging for whole chain --- config.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ run_chain.py | 7 +++++++ 2 files changed, 54 insertions(+) diff --git a/config.py b/config.py index fc5e36ae..b0987f3c 100644 --- a/config.py +++ b/config.py @@ -56,6 +56,9 @@ def __init__(self, casename): # User-defined attributes from config file self.load_config_file(casename) + # Set case root + self.case_root = self.work_root / self.casename + # Set workflow and async attributes and initiate job ids dict self.set_workflow() @@ -324,6 +327,50 @@ def create_vars_from_dicts(self, dct=None, key=None): else: setattr(self, subkey, v) + def log_job_status(self, job, status, launch_time, duration=None): + log_file = self.case_root / "chain_status.log" + + # Check if the header exists, if not, create it + if not log_file.is_file(): + header = "Name ID Status Time Duration\n" + with open(log_file, 'w') as f: + f.write(header) + + # Log the job information + if job == 'chain': + if duration is not None: + duration = self.format_duration(duration) + job_id = '' + else: + job_id = self.job_id + + launch_time = launch_time.strftime("%a %b %d %H:%M:%S %Z %Y") + if status == 'FINISH': + log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<28} {duration}\n" + else: + log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<28}\n" + + with open(log_file, 'a') as f: + f.write(log_entry) + + def format_duration(self, duration): + """ + Format a duration represented by a datetime.timedelta object into a human-readable string. + + Parameters: + - duration (datetime.timedelta): The duration to be formatted. + + Returns: + - str: A string representing the formatted duration in the "0d 0h 0m 0s" format. + """ + seconds = duration.total_seconds() + days, remainder = divmod(seconds, 86400) + hours, remainder = divmod(remainder, 3600) + minutes, seconds = divmod(remainder, 60) + + formatted_duration = f"{int(days)}d {int(hours)}h {int(minutes)}m {int(seconds)}s" + return formatted_duration + def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" diff --git a/run_chain.py b/run_chain.py index 1a2e8e3e..6149e56d 100755 --- a/run_chain.py +++ b/run_chain.py @@ -432,6 +432,10 @@ def main(): f"Starting chain for case {casename} and workflow {cfg.workflow_name}" ) + launch_time = datetime.now() + tools.create_dir(cfg.case_root, "case_root") + cfg.log_job_status('chain', 'START', launch_time) + # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: if hasattr(cfg, 'spinup'): @@ -448,6 +452,9 @@ def main(): cfg.enddate_sim = cfg.enddate run_chunk(cfg=cfg, force=args.force, resume=args.resume) + end_time = datetime.now() + duration_seconds = (end_time - launch_time).total_seconds() + cfg.log_job_status('chain', 'START', launch_time, duration_seconds) print('>>> Finished the processing chain successfully <<<') From 7c1a58d408209aead90ab48db31193b524da3f24 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 20 Dec 2023 12:52:21 +0000 Subject: [PATCH 041/245] GitHub Action: Apply Pep8-formatting --- config.py | 2 +- run_chain.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index b0987f3c..1ddd70f1 100644 --- a/config.py +++ b/config.py @@ -341,7 +341,7 @@ def log_job_status(self, job, status, launch_time, duration=None): if duration is not None: duration = self.format_duration(duration) job_id = '' - else: + else: job_id = self.job_id launch_time = launch_time.strftime("%a %b %d %H:%M:%S %Z %Y") diff --git a/run_chain.py b/run_chain.py index 6149e56d..cc0854b9 100755 --- a/run_chain.py +++ b/run_chain.py @@ -267,8 +267,8 @@ def run_chunk(cfg, force, resume): job, cfg.job_id) logging.exception(subject) if cfg.user_mail: - message = tools.prepare_message(cfg.log_working_dir / - job) + message = tools.prepare_message( + cfg.log_working_dir / job) logging.info('Sending log file to %s' % cfg.user_mail) tools.send_mail(cfg.user_mail, subject, message) @@ -279,7 +279,8 @@ def run_chunk(cfg, force, resume): subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( job, cfg.job_id) if cfg.user_mail: - message = tools.prepare_message(cfg.log_working_dir / job) + message = tools.prepare_message(cfg.log_working_dir / + job) logging.info('Sending log file to %s' % cfg.user_mail) tools.send_mail(cfg.user_mail, subject, message) raise RuntimeError(subject) From 59393b3e969f2e1aeb8100b48a65a00b74ca71fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 20 Dec 2023 17:18:52 +0100 Subject: [PATCH 042/245] Add logging for jobs --- config.py | 29 +++++++++++++++++++++++------ run_chain.py | 11 ++++++++--- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/config.py b/config.py index b0987f3c..8cdda325 100644 --- a/config.py +++ b/config.py @@ -316,7 +316,6 @@ def create_vars_from_dicts(self, dct=None, key=None): new instance attributes by concatenating the original attribute name and the dictionary key, and assigns the corresponding values. """ - if dct is None: self.create_vars_from_dicts(dct=vars(self).copy()) else: @@ -328,11 +327,29 @@ def create_vars_from_dicts(self, dct=None, key=None): setattr(self, subkey, v) def log_job_status(self, job, status, launch_time, duration=None): + """ + Log the status of a job in a chain to a file. + + Parameters: + - job (str): The type of job, either 'chain' or a specific job name. + - status (str): The status of the job, e.g., 'FINISH', 'ERROR', etc. + - launch_time (datetime.datetime): The timestamp when the job was launched or finished. + - duration (datetime.timedelta, optional): The duration of the job. Default is None. + + The function logs the job information to a file named 'chain_status.log' in the case root directory. + If the log file doesn't exist, it creates a header with the column names. + The logged entry includes the job type, job ID (if applicable), status, launch time, + and optionally, the duration if provided. + + Example: + - log_job_status('chain', 'FINISH', datetime.datetime.now(), '00:15:30') + - log_job_status('task_1', 'ERROR', datetime.datetime(2023, 12, 20, 8, 30, 15)) + """ log_file = self.case_root / "chain_status.log" # Check if the header exists, if not, create it if not log_file.is_file(): - header = "Name ID Status Time Duration\n" + header = "Name ID Status Time Duration\n" with open(log_file, 'w') as f: f.write(header) @@ -344,11 +361,11 @@ def log_job_status(self, job, status, launch_time, duration=None): else: job_id = self.job_id - launch_time = launch_time.strftime("%a %b %d %H:%M:%S %Z %Y") - if status == 'FINISH': - log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<28} {duration}\n" + launch_time = launch_time.strftime("%a %b %d %Y %H:%M:%S") + if status == 'FINISH' and duration: + log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<24} {duration}\n" else: - log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<28}\n" + log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<24}\n" with open(log_file, 'a') as f: f.write(log_entry) diff --git a/run_chain.py b/run_chain.py index 6149e56d..06706cd8 100755 --- a/run_chain.py +++ b/run_chain.py @@ -206,10 +206,15 @@ def run_chunk(cfg, force, resume): else: # Submit job and process logfile print(f' └── Process "{job}" for chunk "{cfg.job_id}"') - getattr(jobs, job).main(cfg) logfile = cfg.log_working_dir / job logfile_finish = cfg.log_finished_dir / job tools.change_logfile(logfile) + job_launch_time = datetime.now() + cfg.log_job_status(job, 'START', job_launch_time) + getattr(jobs, job).main(cfg) + job_end_time = datetime.now() + job_duration = job_end_time - job_launch_time + cfg.log_job_status('chain', 'FINISH', job_end_time, job_duration) shutil.copy(logfile, logfile_finish) # wait for previous chunk to be done @@ -453,8 +458,8 @@ def main(): run_chunk(cfg=cfg, force=args.force, resume=args.resume) end_time = datetime.now() - duration_seconds = (end_time - launch_time).total_seconds() - cfg.log_job_status('chain', 'START', launch_time, duration_seconds) + duration = end_time - launch_time + cfg.log_job_status('chain', 'FINISH', end_time, duration) print('>>> Finished the processing chain successfully <<<') From d2fa4107f942697cea6c98bce46d8d8fb6bff77b Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 20 Dec 2023 16:19:25 +0000 Subject: [PATCH 043/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 3957896b..dd5107d4 100755 --- a/run_chain.py +++ b/run_chain.py @@ -214,7 +214,8 @@ def run_chunk(cfg, force, resume): getattr(jobs, job).main(cfg) job_end_time = datetime.now() job_duration = job_end_time - job_launch_time - cfg.log_job_status('chain', 'FINISH', job_end_time, job_duration) + cfg.log_job_status('chain', 'FINISH', job_end_time, + job_duration) shutil.copy(logfile, logfile_finish) # wait for previous chunk to be done From 47da701c43170d7d40c0b18aec7f12f303ff1637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 20 Dec 2023 17:29:09 +0100 Subject: [PATCH 044/245] Small fix --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index dd5107d4..aa836d41 100755 --- a/run_chain.py +++ b/run_chain.py @@ -214,7 +214,7 @@ def run_chunk(cfg, force, resume): getattr(jobs, job).main(cfg) job_end_time = datetime.now() job_duration = job_end_time - job_launch_time - cfg.log_job_status('chain', 'FINISH', job_end_time, + cfg.log_job_status(job, 'FINISH', job_end_time, job_duration) shutil.copy(logfile, logfile_finish) From 1c65c5b5301edb00cf95420c566e26c00dc30ad6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 20 Dec 2023 16:29:36 +0000 Subject: [PATCH 045/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index aa836d41..2ff78924 100755 --- a/run_chain.py +++ b/run_chain.py @@ -214,8 +214,7 @@ def run_chunk(cfg, force, resume): getattr(jobs, job).main(cfg) job_end_time = datetime.now() job_duration = job_end_time - job_launch_time - cfg.log_job_status(job, 'FINISH', job_end_time, - job_duration) + cfg.log_job_status(job, 'FINISH', job_end_time, job_duration) shutil.copy(logfile, logfile_finish) # wait for previous chunk to be done From b0dd12a2edaa55e5d00325c9c156ea0778f5cd76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 21 Dec 2023 10:26:52 +0100 Subject: [PATCH 046/245] Format duration --- config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index e4f710d9..dd5eb2d5 100644 --- a/config.py +++ b/config.py @@ -353,14 +353,17 @@ def log_job_status(self, job, status, launch_time, duration=None): with open(log_file, 'w') as f: f.write(header) - # Log the job information + # Format duration and job_id if job == 'chain': if duration is not None: duration = self.format_duration(duration) job_id = '' else: + if duration is not None: + duration = f"{str(int(duration.total_seconds()))} s" job_id = self.job_id + # Log the job information launch_time = launch_time.strftime("%a %b %d %Y %H:%M:%S") if status == 'FINISH' and duration: log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<24} {duration}\n" From 54b21a69b9a9d647d953b67775b3a2dcf83d9023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 21 Dec 2023 13:50:59 +0100 Subject: [PATCH 047/245] Remove --wait from submit function --- config.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index dd5eb2d5..5840da79 100644 --- a/config.py +++ b/config.py @@ -420,13 +420,9 @@ def get_dep_ids(self, job_name, add_dep=None): dep_id_list.extend(dep_id) return dep_id_list - def get_dep_cmd(self, job_name, add_dep=None, wait=False): + def get_dep_cmd(self, job_name, add_dep=None): """Generate the part of the sbatch command that sepcifies dependencies for job_name.""" - - if wait: - # forced wait - return '--wait' - elif self.is_async: + if self.is_async: # async case if dep_ids := self.get_dep_ids(job_name, add_dep=add_dep): dep_str = ':'.join(map(str, dep_ids)) @@ -439,12 +435,12 @@ def get_dep_cmd(self, job_name, add_dep=None, wait=False): # sequential case return '--wait' - def submit(self, job_name, script, add_dep=None, wait=False): + def submit(self, job_name, script, add_dep=None): """Submit job with dependencies""" script_path = Path(script) sbatch_cmd = ['sbatch', '--parsable'] - if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep, wait=wait): + if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): sbatch_cmd.append(dep_cmd) sbatch_cmd.append(script_path.name) From 5769f8a26bc661314bc171249bbb503e5428a377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 21 Dec 2023 14:47:06 +0100 Subject: [PATCH 048/245] Let geosp be a separate job --- cases/icon-async-test/config.yaml | 8 +- jobs/__init__.py | 1 + jobs/geosp.py | 62 ++++++++++ jobs/prepare_art-oem.py | 181 ++++++++++++++++++++++++++++++ jobs/prepare_art.py | 181 ++++++++++++++++++++++++++++++ jobs/prepare_data.py | 169 ++-------------------------- workflows.yaml | 6 +- 7 files changed, 437 insertions(+), 171 deletions(-) create mode 100644 jobs/geosp.py create mode 100644 jobs/prepare_art-oem.py create mode 100644 jobs/prepare_art.py diff --git a/cases/icon-async-test/config.yaml b/cases/icon-async-test/config.yaml index 5d96ca6c..1000bebf 100644 --- a/cases/icon-async-test/config.yaml +++ b/cases/icon-async-test/config.yaml @@ -1,12 +1,6 @@ # Configuration file for the 'icon-async-test' case with ICON -workflow: - jobs: ['prepare_data', 'icon'] - features: ['restart'] - dependencies: - icon: - current: ['prepare_data'] - previous: ['icon'] +workflow: icon-async constraint: gpu run_on: cpu compute_queue: normal diff --git a/jobs/__init__.py b/jobs/__init__.py index a269fbf3..b74b46f3 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -5,6 +5,7 @@ from . import check_output from . import cosmo from . import emissions +from . import geosp from . import icon from . import int2lm from . import obs_nudging diff --git a/jobs/geosp.py b/jobs/geosp.py new file mode 100644 index 00000000..d01d8f38 --- /dev/null +++ b/jobs/geosp.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +from pathlib import Path +import logging +from datetime import timedelta +import xarray as xr +import numpy as np +from . import tools, prepare_data + +def main(cfg): + """ + Add GEOSP + """ + prepare_data.set_cfg_variables(cfg) + + #----------------------------------------------------- + # Add GEOSP to all meteo files + #----------------------------------------------------- + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + # Specify file names + geosp_filename = time.replace( + hour=0).strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) + src_filename = time.strftime( + cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' + src_file = os.path.join(cfg.icon_input_icbc, src_filename) + merged_filename = time.strftime( + cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, + merged_filename) + + # Copy GEOSP file from last run if not present + if not os.path.exists(geosp_file): + geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, + geosp_filename) + tools.copy_file(geosp_src_file, + cfg.icon_input_icbc, + output_log=True) + + # Load GEOSP data array as da_geosp at time 00: + ds = xr.open_dataset(src_file) + ds_geosp = xr.open_dataset(geosp_file) + da_geosp = ds_geosp['GEOSP'] + + # Merge GEOSP-dataset with other timesteps + if (time.hour != 0): + # Change values of time dimension to current time + da_geosp = da_geosp.assign_coords( + time=[np.datetime64(time)]) + # Merge GEOSP into temporary file + ds_merged = xr.merge([ds, da_geosp]) + ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Logging info for merging GEOSP + logging.info("Added GEOSP to file {}".format(merged_file)) + # Rename file to get original file name + tools.rename_file(merged_file, src_file) diff --git a/jobs/prepare_art-oem.py b/jobs/prepare_art-oem.py new file mode 100644 index 00000000..558a9048 --- /dev/null +++ b/jobs/prepare_art-oem.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +from pathlib import Path +import logging +import shutil +import subprocess +from datetime import timedelta +import xarray as xr +import numpy as np +from . import tools, prepare_data +from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond +from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging +from calendar import monthrange + +def main(cfg): + """ + Add GEOSP + """ + prepare_data.set_cfg_variables(cfg) + + #----------------------------------------------------- + # Add GEOSP to all meteo files + #----------------------------------------------------- + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + # Specify file names + geosp_filename = time.replace( + hour=0).strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) + src_filename = time.strftime( + cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' + src_file = os.path.join(cfg.icon_input_icbc, src_filename) + merged_filename = time.strftime( + cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, + merged_filename) + + # Copy GEOSP file from last run if not present + if not os.path.exists(geosp_file): + geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, + geosp_filename) + tools.copy_file(geosp_src_file, + cfg.icon_input_icbc, + output_log=True) + + # Load GEOSP data array as da_geosp at time 00: + ds = xr.open_dataset(src_file) + ds_geosp = xr.open_dataset(geosp_file) + da_geosp = ds_geosp['GEOSP'] + + # Merge GEOSP-dataset with other timesteps + if (time.hour != 0): + # Change values of time dimension to current time + da_geosp = da_geosp.assign_coords( + time=[np.datetime64(time)]) + # Merge GEOSP into temporary file + ds_merged = xr.merge([ds, da_geosp]) + ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Logging info for merging GEOSP + logging.info("Added GEOSP to file {}".format(merged_file)) + # Rename file to get original file name + tools.rename_file(merged_file, src_file) + + #----------------------------------------------------- + # Add Q (copy of QV) and/or PS to initial file + #----------------------------------------------------- + if cfg.workflow_name.startswith('icon-art'): + meteo_file = os.path.join( + cfg.icon_input_icbc, + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + + '.nc') + if os.path.isfile(meteo_file): + merged_file = os.path.join( + cfg.icon_input_icbc, + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + + '_merged.nc') + ds = xr.open_dataset(meteo_file) + merging = False + if 'PS' not in ds: + if 'LNPS' not in ds: + raise KeyError( + f"'LNPS' must be found in the initial conditions file {meteo_file}" + ) + merging = True + ds['PS'] = ds['LNPS'] + ds['PS'].attrs = ds['LNPS'].attrs + ds['PS'] = np.exp(ds['PS']) + ds['PS'] = ds['PS'].squeeze(dim='lev_2') + ds['PS'].attrs["long_name"] = 'surface pressure' + ds['PS'].attrs['units'] = 'Pa' + logging.info(f"Added PS to file {meteo_file}") + if 'Q' not in ds: + merging = True + ds['Q'] = ds['QV'] + logging.info(f"Added Q to file {meteo_file}") + if merging: + ds.to_netcdf(merged_file) + tools.rename_file(merged_file, meteo_file) + + #----------------------------------------------------- + # In case of OEM: merge chem tracers with meteo-files + #----------------------------------------------------- + if cfg.workflow_name == 'icon-art-oem': + for time in tools.iter_hours(cfg.startdate_sim, + cfg.enddate_sim, + cfg.meteo['inc']): + if time == cfg.startdate_sim: + #------------ + # Merge IC: + #------------ + meteo_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '.nc') + if os.path.isfile(meteo_file): + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem['nameformat']) + '.nc') + merged_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + + '_merged.nc') + ds_meteo = xr.open_dataset(meteo_file) + ds_chem = xr.open_dataset(chem_file) + # LNPS --> PS + ds_chem['PS'] = ds_chem['LNPS'] + ds_chem['PS'].attrs = ds_chem['LNPS'].attrs + ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') + ds_chem['PS'].attrs[ + "long_name"] = 'surface pressure' + # merge: + ds_merged = xr.merge([ds_meteo, ds_chem], + compat="override") + #ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Rename file to get original file name + tools.rename_file(merged_file, meteo_file) + tools.remove_file(chem_file) + logging.info( + "Added chemical tracer to file {}".format( + merged_file)) + + #------------ + # Merge LBC: + #------------ + meteo_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc') + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem_nameformat) + '_lbc.nc') + merged_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc') + ds_meteo = xr.open_dataset(meteo_file) + ds_chem = xr.open_dataset(chem_file) + # LNPS --> PS + ds_chem['PS'] = ds_chem['LNPS'] + ds_chem['PS'].attrs = ds_chem['LNPS'].attrs + ds_chem['PS'].attrs["long_name"] = 'surface pressure' + ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] + # merge: + ds_merged = xr.merge([ds_meteo, ds_chem], + compat="override") + #ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Rename file to get original file name + tools.rename_file(merged_file, meteo_file) + tools.remove_file(chem_file) + logging.info( + "Added chemical tracer to file {}".format(merged_file)) diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py new file mode 100644 index 00000000..558a9048 --- /dev/null +++ b/jobs/prepare_art.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +from pathlib import Path +import logging +import shutil +import subprocess +from datetime import timedelta +import xarray as xr +import numpy as np +from . import tools, prepare_data +from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond +from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging +from calendar import monthrange + +def main(cfg): + """ + Add GEOSP + """ + prepare_data.set_cfg_variables(cfg) + + #----------------------------------------------------- + # Add GEOSP to all meteo files + #----------------------------------------------------- + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + # Specify file names + geosp_filename = time.replace( + hour=0).strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) + src_filename = time.strftime( + cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' + src_file = os.path.join(cfg.icon_input_icbc, src_filename) + merged_filename = time.strftime( + cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, + merged_filename) + + # Copy GEOSP file from last run if not present + if not os.path.exists(geosp_file): + geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, + geosp_filename) + tools.copy_file(geosp_src_file, + cfg.icon_input_icbc, + output_log=True) + + # Load GEOSP data array as da_geosp at time 00: + ds = xr.open_dataset(src_file) + ds_geosp = xr.open_dataset(geosp_file) + da_geosp = ds_geosp['GEOSP'] + + # Merge GEOSP-dataset with other timesteps + if (time.hour != 0): + # Change values of time dimension to current time + da_geosp = da_geosp.assign_coords( + time=[np.datetime64(time)]) + # Merge GEOSP into temporary file + ds_merged = xr.merge([ds, da_geosp]) + ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Logging info for merging GEOSP + logging.info("Added GEOSP to file {}".format(merged_file)) + # Rename file to get original file name + tools.rename_file(merged_file, src_file) + + #----------------------------------------------------- + # Add Q (copy of QV) and/or PS to initial file + #----------------------------------------------------- + if cfg.workflow_name.startswith('icon-art'): + meteo_file = os.path.join( + cfg.icon_input_icbc, + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + + '.nc') + if os.path.isfile(meteo_file): + merged_file = os.path.join( + cfg.icon_input_icbc, + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + + '_merged.nc') + ds = xr.open_dataset(meteo_file) + merging = False + if 'PS' not in ds: + if 'LNPS' not in ds: + raise KeyError( + f"'LNPS' must be found in the initial conditions file {meteo_file}" + ) + merging = True + ds['PS'] = ds['LNPS'] + ds['PS'].attrs = ds['LNPS'].attrs + ds['PS'] = np.exp(ds['PS']) + ds['PS'] = ds['PS'].squeeze(dim='lev_2') + ds['PS'].attrs["long_name"] = 'surface pressure' + ds['PS'].attrs['units'] = 'Pa' + logging.info(f"Added PS to file {meteo_file}") + if 'Q' not in ds: + merging = True + ds['Q'] = ds['QV'] + logging.info(f"Added Q to file {meteo_file}") + if merging: + ds.to_netcdf(merged_file) + tools.rename_file(merged_file, meteo_file) + + #----------------------------------------------------- + # In case of OEM: merge chem tracers with meteo-files + #----------------------------------------------------- + if cfg.workflow_name == 'icon-art-oem': + for time in tools.iter_hours(cfg.startdate_sim, + cfg.enddate_sim, + cfg.meteo['inc']): + if time == cfg.startdate_sim: + #------------ + # Merge IC: + #------------ + meteo_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '.nc') + if os.path.isfile(meteo_file): + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem['nameformat']) + '.nc') + merged_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + + '_merged.nc') + ds_meteo = xr.open_dataset(meteo_file) + ds_chem = xr.open_dataset(chem_file) + # LNPS --> PS + ds_chem['PS'] = ds_chem['LNPS'] + ds_chem['PS'].attrs = ds_chem['LNPS'].attrs + ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') + ds_chem['PS'].attrs[ + "long_name"] = 'surface pressure' + # merge: + ds_merged = xr.merge([ds_meteo, ds_chem], + compat="override") + #ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Rename file to get original file name + tools.rename_file(merged_file, meteo_file) + tools.remove_file(chem_file) + logging.info( + "Added chemical tracer to file {}".format( + merged_file)) + + #------------ + # Merge LBC: + #------------ + meteo_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc') + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem_nameformat) + '_lbc.nc') + merged_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc') + ds_meteo = xr.open_dataset(meteo_file) + ds_chem = xr.open_dataset(chem_file) + # LNPS --> PS + ds_chem['PS'] = ds_chem['LNPS'] + ds_chem['PS'].attrs = ds_chem['LNPS'].attrs + ds_chem['PS'].attrs["long_name"] = 'surface pressure' + ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] + # merge: + ds_merged = xr.merge([ds_meteo, ds_chem], + compat="override") + #ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Rename file to get original file name + tools.rename_file(merged_file, meteo_file) + tools.remove_file(chem_file) + logging.info( + "Added chemical tracer to file {}".format(merged_file)) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 14e09bd6..b1d9edbf 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -134,7 +134,7 @@ def main(cfg): with (script := cfg.icon_work / 'copy_input.job').open('w') as f: f.write('\n'.join(script_lines)) - cfg.submit('prepare_data', script, wait=True) + _, copy_id = cfg.submit('prepare_data', script) if cfg.workflow_name == 'icon-art-global': async_error(cfg, part='global ICON-ART') @@ -325,6 +325,7 @@ def main(cfg): #----------------------------------------------------- # Write and submit runscripts #----------------------------------------------------- + icontools_id = None for runscript in cfg.icontools_runjobs: with open(os.path.join(cfg.case_path, runscript)) as input_file: @@ -342,167 +343,11 @@ def main(cfg): # Submitting icontools runscripts sequentially logging.info(f" Starting icontools runscript {runscript}.") - cfg.submit(runscript, runscript_path, wait=True) - - #----------------------------------------------------- - # Add GEOSP to all meteo files - #----------------------------------------------------- - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - # Specify file names - geosp_filename = time.replace( - hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime( - cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' - src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime( - cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, - merged_filename) - - # Copy GEOSP file from last run if not present - if not os.path.exists(geosp_file): - geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) - tools.copy_file(geosp_src_file, - cfg.icon_input_icbc, - output_log=True) - - # Load GEOSP data array as da_geosp at time 00: - ds = xr.open_dataset(src_file) - ds_geosp = xr.open_dataset(geosp_file) - da_geosp = ds_geosp['GEOSP'] - - # Merge GEOSP-dataset with other timesteps - if (time.hour != 0): - # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords( - time=[np.datetime64(time)]) - # Merge GEOSP into temporary file - ds_merged = xr.merge([ds, da_geosp]) - ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Logging info for merging GEOSP - logging.info("Added GEOSP to file {}".format(merged_file)) - # Rename file to get original file name - tools.rename_file(merged_file, src_file) - - #----------------------------------------------------- - # Add Q (copy of QV) and/or PS to initial file - #----------------------------------------------------- - if cfg.workflow_name.startswith('icon-art'): - meteo_file = os.path.join( - cfg.icon_input_icbc, - cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '.nc') - if os.path.isfile(meteo_file): - merged_file = os.path.join( - cfg.icon_input_icbc, - cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') - ds = xr.open_dataset(meteo_file) - merging = False - if 'PS' not in ds: - if 'LNPS' not in ds: - raise KeyError( - f"'LNPS' must be found in the initial conditions file {meteo_file}" - ) - merging = True - ds['PS'] = ds['LNPS'] - ds['PS'].attrs = ds['LNPS'].attrs - ds['PS'] = np.exp(ds['PS']) - ds['PS'] = ds['PS'].squeeze(dim='lev_2') - ds['PS'].attrs["long_name"] = 'surface pressure' - ds['PS'].attrs['units'] = 'Pa' - logging.info(f"Added PS to file {meteo_file}") - if 'Q' not in ds: - merging = True - ds['Q'] = ds['QV'] - logging.info(f"Added Q to file {meteo_file}") - if merging: - ds.to_netcdf(merged_file) - tools.rename_file(merged_file, meteo_file) - - #----------------------------------------------------- - # In case of OEM: merge chem tracers with meteo-files - #----------------------------------------------------- - if cfg.workflow_name == 'icon-art-oem': - for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - cfg.meteo['inc']): - if time == cfg.startdate_sim: - #------------ - # Merge IC: - #------------ - meteo_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') - if os.path.isfile(meteo_file): - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem['nameformat']) + '.nc') - merged_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') - ds_meteo = xr.open_dataset(meteo_file) - ds_chem = xr.open_dataset(chem_file) - # LNPS --> PS - ds_chem['PS'] = ds_chem['LNPS'] - ds_chem['PS'].attrs = ds_chem['LNPS'].attrs - ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') - ds_chem['PS'].attrs[ - "long_name"] = 'surface pressure' - # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") - #ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Rename file to get original file name - tools.rename_file(merged_file, meteo_file) - tools.remove_file(chem_file) - logging.info( - "Added chemical tracer to file {}".format( - merged_file)) - - #------------ - # Merge LBC: - #------------ - meteo_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc') - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem_nameformat) + '_lbc.nc') - merged_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc') - ds_meteo = xr.open_dataset(meteo_file) - ds_chem = xr.open_dataset(chem_file) - # LNPS --> PS - ds_chem['PS'] = ds_chem['LNPS'] - ds_chem['PS'].attrs = ds_chem['LNPS'].attrs - ds_chem['PS'].attrs["long_name"] = 'surface pressure' - ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] - # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") - #ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Rename file to get original file name - tools.rename_file(merged_file, meteo_file) - tools.remove_file(chem_file) - logging.info( - "Added chemical tracer to file {}".format(merged_file)) + if icontools_id: + dependencies = (copy_id, icontools_id) + else: + dependencies = copy_id + _, icontools_id = cfg.submit(runscript, runscript_path, add_dep=dependencies) # If COSMO (and not ICON): else: diff --git a/workflows.yaml b/workflows.yaml index 45130424..a451b072 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -10,11 +10,13 @@ cosmo-art: 'int2lm', 'cosmo', 'post_cosmo'] features: ['nesting', 'spinup'] icon-async: - jobs: ['prepare_data', 'icon'] + jobs: ['prepare_data', 'geosp', 'icon'] features: ['restart'] dependencies: + geosp: + current: ['prepare_data'] icon: - current: ['prepare_data'] + current: ['prepare_data', 'geosp'] previous: ['icon'] icon: jobs: ['prepare_data', 'icon'] From df4fb2cb2166d1e502518a3fe95b670429ed5e50 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 21 Dec 2023 13:47:30 +0000 Subject: [PATCH 049/245] GitHub Action: Apply Pep8-formatting --- jobs/geosp.py | 22 ++++++++-------- jobs/prepare_art-oem.py | 56 ++++++++++++++++++----------------------- jobs/prepare_art.py | 56 ++++++++++++++++++----------------------- jobs/prepare_data.py | 4 ++- 4 files changed, 61 insertions(+), 77 deletions(-) diff --git a/jobs/geosp.py b/jobs/geosp.py index d01d8f38..ba8c93a5 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -9,6 +9,7 @@ import numpy as np from . import tools, prepare_data + def main(cfg): """ Add GEOSP @@ -19,25 +20,23 @@ def main(cfg): # Add GEOSP to all meteo files #----------------------------------------------------- for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): # Specify file names geosp_filename = time.replace( hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' + cfg.meteo['nameformat']) + '_lbc.nc' geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime( - cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' + src_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime( - cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, - merged_filename) + merged_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) # Copy GEOSP file from last run if not present if not os.path.exists(geosp_file): geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) + geosp_filename) tools.copy_file(geosp_src_file, cfg.icon_input_icbc, output_log=True) @@ -50,8 +49,7 @@ def main(cfg): # Merge GEOSP-dataset with other timesteps if (time.hour != 0): # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords( - time=[np.datetime64(time)]) + da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) # Merge GEOSP into temporary file ds_merged = xr.merge([ds, da_geosp]) ds_merged.attrs = ds.attrs diff --git a/jobs/prepare_art-oem.py b/jobs/prepare_art-oem.py index 558a9048..f027db36 100644 --- a/jobs/prepare_art-oem.py +++ b/jobs/prepare_art-oem.py @@ -14,6 +14,7 @@ from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging from calendar import monthrange + def main(cfg): """ Add GEOSP @@ -24,25 +25,23 @@ def main(cfg): # Add GEOSP to all meteo files #----------------------------------------------------- for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): # Specify file names geosp_filename = time.replace( hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' + cfg.meteo['nameformat']) + '_lbc.nc' geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime( - cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' + src_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime( - cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, - merged_filename) + merged_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) # Copy GEOSP file from last run if not present if not os.path.exists(geosp_file): geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) + geosp_filename) tools.copy_file(geosp_src_file, cfg.icon_input_icbc, output_log=True) @@ -55,8 +54,7 @@ def main(cfg): # Merge GEOSP-dataset with other timesteps if (time.hour != 0): # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords( - time=[np.datetime64(time)]) + da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) # Merge GEOSP into temporary file ds_merged = xr.merge([ds, da_geosp]) ds_merged.attrs = ds.attrs @@ -73,13 +71,12 @@ def main(cfg): meteo_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '.nc') + cfg.meteo['nameformat']) + '.nc') if os.path.isfile(meteo_file): merged_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + + cfg.meteo['nameformat']) + '_merged.nc') ds = xr.open_dataset(meteo_file) merging = False @@ -108,9 +105,8 @@ def main(cfg): # In case of OEM: merge chem tracers with meteo-files #----------------------------------------------------- if cfg.workflow_name == 'icon-art-oem': - for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - cfg.meteo['inc']): + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): if time == cfg.startdate_sim: #------------ # Merge IC: @@ -118,7 +114,7 @@ def main(cfg): meteo_file = os.path.join( cfg.icon_input_icbc, time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') + cfg.meteo['nameformat']) + '.nc') if os.path.isfile(meteo_file): chem_file = os.path.join( cfg.icon_input_icbc, cfg.chem['prefix'] + @@ -126,42 +122,39 @@ def main(cfg): merged_file = os.path.join( cfg.icon_input_icbc, time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') + cfg.meteo['nameformat']) + '_merged.nc') ds_meteo = xr.open_dataset(meteo_file) ds_chem = xr.open_dataset(chem_file) # LNPS --> PS ds_chem['PS'] = ds_chem['LNPS'] ds_chem['PS'].attrs = ds_chem['LNPS'].attrs ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') - ds_chem['PS'].attrs[ - "long_name"] = 'surface pressure' + ds_chem['PS'].attrs["long_name"] = 'surface pressure' # merge: ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") + compat="override") #ds_merged.attrs = ds.attrs ds_merged.to_netcdf(merged_file) # Rename file to get original file name tools.rename_file(merged_file, meteo_file) tools.remove_file(chem_file) logging.info( - "Added chemical tracer to file {}".format( - merged_file)) + "Added chemical tracer to file {}".format(merged_file)) #------------ # Merge LBC: #------------ meteo_file = os.path.join( cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc') + time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + + '_lbc.nc') chem_file = os.path.join( cfg.icon_input_icbc, cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat) + '_lbc.nc') merged_file = os.path.join( cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc') + time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + + '_merged.nc') ds_meteo = xr.open_dataset(meteo_file) ds_chem = xr.open_dataset(chem_file) # LNPS --> PS @@ -170,8 +163,7 @@ def main(cfg): ds_chem['PS'].attrs["long_name"] = 'surface pressure' ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") + ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") #ds_merged.attrs = ds.attrs ds_merged.to_netcdf(merged_file) # Rename file to get original file name diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 558a9048..f027db36 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -14,6 +14,7 @@ from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging from calendar import monthrange + def main(cfg): """ Add GEOSP @@ -24,25 +25,23 @@ def main(cfg): # Add GEOSP to all meteo files #----------------------------------------------------- for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): # Specify file names geosp_filename = time.replace( hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' + cfg.meteo['nameformat']) + '_lbc.nc' geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime( - cfg.meteo['prefix'] + cfg.meteo['nameformat']) + '_lbc.nc' + src_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime( - cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, - merged_filename) + merged_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) # Copy GEOSP file from last run if not present if not os.path.exists(geosp_file): geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) + geosp_filename) tools.copy_file(geosp_src_file, cfg.icon_input_icbc, output_log=True) @@ -55,8 +54,7 @@ def main(cfg): # Merge GEOSP-dataset with other timesteps if (time.hour != 0): # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords( - time=[np.datetime64(time)]) + da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) # Merge GEOSP into temporary file ds_merged = xr.merge([ds, da_geosp]) ds_merged.attrs = ds.attrs @@ -73,13 +71,12 @@ def main(cfg): meteo_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '.nc') + cfg.meteo['nameformat']) + '.nc') if os.path.isfile(meteo_file): merged_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + + cfg.meteo['nameformat']) + '_merged.nc') ds = xr.open_dataset(meteo_file) merging = False @@ -108,9 +105,8 @@ def main(cfg): # In case of OEM: merge chem tracers with meteo-files #----------------------------------------------------- if cfg.workflow_name == 'icon-art-oem': - for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - cfg.meteo['inc']): + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): if time == cfg.startdate_sim: #------------ # Merge IC: @@ -118,7 +114,7 @@ def main(cfg): meteo_file = os.path.join( cfg.icon_input_icbc, time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') + cfg.meteo['nameformat']) + '.nc') if os.path.isfile(meteo_file): chem_file = os.path.join( cfg.icon_input_icbc, cfg.chem['prefix'] + @@ -126,42 +122,39 @@ def main(cfg): merged_file = os.path.join( cfg.icon_input_icbc, time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') + cfg.meteo['nameformat']) + '_merged.nc') ds_meteo = xr.open_dataset(meteo_file) ds_chem = xr.open_dataset(chem_file) # LNPS --> PS ds_chem['PS'] = ds_chem['LNPS'] ds_chem['PS'].attrs = ds_chem['LNPS'].attrs ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') - ds_chem['PS'].attrs[ - "long_name"] = 'surface pressure' + ds_chem['PS'].attrs["long_name"] = 'surface pressure' # merge: ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") + compat="override") #ds_merged.attrs = ds.attrs ds_merged.to_netcdf(merged_file) # Rename file to get original file name tools.rename_file(merged_file, meteo_file) tools.remove_file(chem_file) logging.info( - "Added chemical tracer to file {}".format( - merged_file)) + "Added chemical tracer to file {}".format(merged_file)) #------------ # Merge LBC: #------------ meteo_file = os.path.join( cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc') + time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + + '_lbc.nc') chem_file = os.path.join( cfg.icon_input_icbc, cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat) + '_lbc.nc') merged_file = os.path.join( cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc') + time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + + '_merged.nc') ds_meteo = xr.open_dataset(meteo_file) ds_chem = xr.open_dataset(chem_file) # LNPS --> PS @@ -170,8 +163,7 @@ def main(cfg): ds_chem['PS'].attrs["long_name"] = 'surface pressure' ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") + ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") #ds_merged.attrs = ds.attrs ds_merged.to_netcdf(merged_file) # Rename file to get original file name diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index b1d9edbf..aeaf9812 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -347,7 +347,9 @@ def main(cfg): dependencies = (copy_id, icontools_id) else: dependencies = copy_id - _, icontools_id = cfg.submit(runscript, runscript_path, add_dep=dependencies) + _, icontools_id = cfg.submit(runscript, + runscript_path, + add_dep=dependencies) # If COSMO (and not ICON): else: From f9a80604791bfb00ddae5d8ec2c5ccbb242e5052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 3 Jan 2024 13:58:31 +0100 Subject: [PATCH 050/245] Add run_chain argument to process specified chunk --- run_chain.py | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/run_chain.py b/run_chain.py index 2ff78924..4e46dc6f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -52,6 +52,18 @@ def parse_arguments(): help=jobs_help, default=None) + chunks_help = ("List of chunks to be executed. A chunk is time" + "frame within the total simulation period." + "It has the format `YYYYMMDDHH_YYYYMMDDHH`." + "If no chunks are given, all chunks within the" + "simulation period will be executed.") + parser.add_argument("-c", + "--chunks", + nargs='*', + dest="chunk_list", + help=chunks_help, + default=None) + force_help = ("Force the processing chain to redo all specified jobs," " even if they have been started already or were finished" " previously. WARNING: Only logfiles get deleted," @@ -312,20 +324,28 @@ def restart_runs(cfg, force, resume): - The function iterates over specified intervals, calling `run_chunk()` for each. - It manages restart settings and logging for each subchain. """ - for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, - cfg.restart_step_hours): - enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) - - if enddate_sim > cfg.enddate: - continue + if not cfg.chunks: + for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, + cfg.restart_step_hours): + enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) + startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") + enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") + job_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" + cfg.chunks.append(job_id) + if enddate_sim > cfg.enddate: + continue + + for job_id in cfg.chunks: + cfg.job_id = job_id + cfg.startdate_sim_yyyymmddhh = job_id[0:10] + cfg.enddate_sim_yyyymmddhh = job_id[-10:] + cfg.startdate_sim = datetime.strptime(cfg.startdate_sim_yyyymmddhh, "%Y%m%d%H") + cfg.enddate_sim = datetime.strptime(cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H") # Set restart variable (only takes effect for ICON) - cfg.lrestart = '.FALSE.' if startdate_sim == cfg.startdate else '.TRUE.' - - print(f"└── Starting chunk with startdate {startdate_sim}") + cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." - cfg.startdate_sim = startdate_sim - cfg.enddate_sim = enddate_sim + print(f"└── Starting chunk with startdate {cfg.startdate_sim}") run_chunk(cfg=cfg, force=force, resume=resume) @@ -434,6 +454,12 @@ def main(): else: cfg.jobs = args.job_list + # Check if chunks are set or if all are used + if args.chunk_list is None: + cfg.chunks = [] + else: + cfg.chunks = args.chunk_list + print( f"Starting chain for case {casename} and workflow {cfg.workflow_name}" ) From bbad6e9c020dabca44594116ae600941daed4963 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 3 Jan 2024 12:58:58 +0000 Subject: [PATCH 051/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/run_chain.py b/run_chain.py index 4e46dc6f..78f5167e 100755 --- a/run_chain.py +++ b/run_chain.py @@ -53,10 +53,10 @@ def parse_arguments(): default=None) chunks_help = ("List of chunks to be executed. A chunk is time" - "frame within the total simulation period." - "It has the format `YYYYMMDDHH_YYYYMMDDHH`." - "If no chunks are given, all chunks within the" - "simulation period will be executed.") + "frame within the total simulation period." + "It has the format `YYYYMMDDHH_YYYYMMDDHH`." + "If no chunks are given, all chunks within the" + "simulation period will be executed.") parser.add_argument("-c", "--chunks", nargs='*', @@ -326,8 +326,9 @@ def restart_runs(cfg, force, resume): """ if not cfg.chunks: for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, - cfg.restart_step_hours): - enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) + cfg.restart_step_hours): + enddate_sim = startdate_sim + timedelta( + hours=cfg.restart_step_hours) startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") job_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" @@ -339,8 +340,10 @@ def restart_runs(cfg, force, resume): cfg.job_id = job_id cfg.startdate_sim_yyyymmddhh = job_id[0:10] cfg.enddate_sim_yyyymmddhh = job_id[-10:] - cfg.startdate_sim = datetime.strptime(cfg.startdate_sim_yyyymmddhh, "%Y%m%d%H") - cfg.enddate_sim = datetime.strptime(cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H") + cfg.startdate_sim = datetime.strptime(cfg.startdate_sim_yyyymmddhh, + "%Y%m%d%H") + cfg.enddate_sim = datetime.strptime(cfg.enddate_sim_yyyymmddhh, + "%Y%m%d%H") # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." From f6e75b362acc57547303ac02e5c30c21ea156d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 3 Jan 2024 14:05:39 +0100 Subject: [PATCH 052/245] Fix datetime comparison in int2lm --- jobs/int2lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index c6b98fdc..519370d0 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -110,7 +110,7 @@ def main(cfg): input_art_filename) # Change of soil model from TERRA to TERRA multi-layer on 2 Aug 2007 - if cfg.startdate_sim < datetime(2007, 8, 2, tzinfo=pytz.UTC): + if cfg.startdate_sim < datetime(2007, 8, 2): multi_layer = ".FALSE." else: multi_layer = ".TRUE." From d885c39ef75e5ca67f5c7b93ea45f3bb7a402115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 11:06:11 +0100 Subject: [PATCH 053/245] Submit all jobs as sbatch --- config.py | 32 ++++++++++++++++++++++++++++++++ run_chain.py | 23 ++++++++++++++++++----- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index 5840da79..9b275960 100644 --- a/config.py +++ b/config.py @@ -465,6 +465,38 @@ def check_submitted_job(self, script, result): raise RuntimeError(f"sbatch returned exitcode {exitcode}") logging.info(f"{script} successfully executed.") + def submit_job_as_sbatch(self, job_name, log_file, add_dep=None): + sbatch_cmd = ['sbatch', '--parsable'] + if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): + sbatch_cmd.append(dep_cmd) + script_lines = [ + '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}"', + f'#SBATCH --nodes=1', f'#SBATCH --output={log_file}', + f'#SBATCH --open-mode=append', + f'#SBATCH --account={self.compute_account}', + f'#SBATCH --partition={self.compute_queue}', + f'#SBATCH --constraint={self.constraint}', '', + 'eval "$(conda shell.bash hook)"', + 'conda activate proc-chain', + f'./run_chain.py {self.casename} -j {job_name} -c {self.job_id} -f -s', + ] + + job_file = self.chain_root / f'{job_name}.sh' + with open(job_file, mode='w') as job_script: + job_script.write('\n'.join(script_lines)) + + sbatch_cmd.append(job_file) + result = subprocess.run(sbatch_cmd, capture_output=True) + job_id = int(result.stdout) + print(f' └── Submitted batch job {job_id}') + + if not job_name in self.job_ids['current']: + self.job_ids['current'][job_name] = [job_id] + else: + self.job_ids['current'][job_name].append(job_id) + + return result, job_id + def wait_for_previous(self): """wait for all jobs of the previous stage to be finished diff --git a/run_chain.py b/run_chain.py index 78f5167e..012fabc3 100755 --- a/run_chain.py +++ b/run_chain.py @@ -64,6 +64,9 @@ def parse_arguments(): help=chunks_help, default=None) + sync_help = ("Force synchronous execution.") + parser.add_argument("-s", "--force-sync", action='store_true', help=sync_help) + force_help = ("Force the processing chain to redo all specified jobs," " even if they have been started already or were finished" " previously. WARNING: Only logfiles get deleted," @@ -216,14 +219,20 @@ def run_chunk(cfg, force, resume): print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') skip = True else: - # Submit job and process logfile print(f' └── Process "{job}" for chunk "{cfg.job_id}"') + + # Logfile settings logfile = cfg.log_working_dir / job logfile_finish = cfg.log_finished_dir / job tools.change_logfile(logfile) job_launch_time = datetime.now() cfg.log_job_status(job, 'START', job_launch_time) - getattr(jobs, job).main(cfg) + + # Submit the job + #getattr(jobs, job).main(cfg) + result, job_id = cfg.submit_job_as_sbatch(job, logfile) + + # Logging job_end_time = datetime.now() job_duration = job_end_time - job_launch_time cfg.log_job_status(job, 'FINISH', job_end_time, job_duration) @@ -443,9 +452,6 @@ def main(): # Set restart step in hours cfg.set_restart_step_hours() - # Print config before duplication of dict variables - cfg.print_config() - # Duplicate variables in the form of _ for better # access within namelist template. # E.g.: cfg.meteo['dir'] -> cfg.meteo_dir @@ -463,6 +469,13 @@ def main(): else: cfg.chunks = args.chunk_list + # Check sync is forced + if args.force_sync: + cfg.is_async = None + + # Print config before chain starts + cfg.print_config() + print( f"Starting chain for case {casename} and workflow {cfg.workflow_name}" ) From 3181262a229039f8717f8a0bcdb10fbea2e5de0e Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 4 Jan 2024 10:06:47 +0000 Subject: [PATCH 054/245] GitHub Action: Apply Pep8-formatting --- config.py | 9 ++++++--- run_chain.py | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 9b275960..2752b69f 100644 --- a/config.py +++ b/config.py @@ -470,12 +470,15 @@ def submit_job_as_sbatch(self, job_name, log_file, add_dep=None): if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): sbatch_cmd.append(dep_cmd) script_lines = [ - '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}"', - f'#SBATCH --nodes=1', f'#SBATCH --output={log_file}', + '#!/usr/bin/env bash', + f'#SBATCH --job-name="{job_name}"', + f'#SBATCH --nodes=1', + f'#SBATCH --output={log_file}', f'#SBATCH --open-mode=append', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', - f'#SBATCH --constraint={self.constraint}', '', + f'#SBATCH --constraint={self.constraint}', + '', 'eval "$(conda shell.bash hook)"', 'conda activate proc-chain', f'./run_chain.py {self.casename} -j {job_name} -c {self.job_id} -f -s', diff --git a/run_chain.py b/run_chain.py index 012fabc3..b323676b 100755 --- a/run_chain.py +++ b/run_chain.py @@ -65,7 +65,10 @@ def parse_arguments(): default=None) sync_help = ("Force synchronous execution.") - parser.add_argument("-s", "--force-sync", action='store_true', help=sync_help) + parser.add_argument("-s", + "--force-sync", + action='store_true', + help=sync_help) force_help = ("Force the processing chain to redo all specified jobs," " even if they have been started already or were finished" From f1c757dd10311a447ab4315d9977bcb95fe15d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 11:31:51 +0100 Subject: [PATCH 055/245] Use existing submit() function and only return job_id --- config.py | 27 ++++++--------------------- jobs/icon.py | 25 +++++++++++++------------ jobs/prepare_data.py | 2 +- run_chain.py | 6 ++++-- 4 files changed, 24 insertions(+), 36 deletions(-) diff --git a/config.py b/config.py index 9b275960..67e12c30 100644 --- a/config.py +++ b/config.py @@ -455,20 +455,13 @@ def submit(self, job_name, script, add_dep=None): else: self.job_ids['current'][job_name].append(job_id) - # If needed internaly in a multi-job task like prepare_data - # Can then be passed as add_dep keyword - return result, job_id - - def check_submitted_job(self, script, result): exitcode = result.returncode if exitcode != 0: raise RuntimeError(f"sbatch returned exitcode {exitcode}") - logging.info(f"{script} successfully executed.") - def submit_job_as_sbatch(self, job_name, log_file, add_dep=None): - sbatch_cmd = ['sbatch', '--parsable'] - if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): - sbatch_cmd.append(dep_cmd) + return job_id + + def create_sbatch_script(self, job_name, log_file): script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}"', f'#SBATCH --nodes=1', f'#SBATCH --output={log_file}', @@ -476,26 +469,18 @@ def submit_job_as_sbatch(self, job_name, log_file, add_dep=None): f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', '', + f'cd {self.chain_src_dir}', 'eval "$(conda shell.bash hook)"', 'conda activate proc-chain', f'./run_chain.py {self.casename} -j {job_name} -c {self.job_id} -f -s', + '', ] job_file = self.chain_root / f'{job_name}.sh' with open(job_file, mode='w') as job_script: job_script.write('\n'.join(script_lines)) - sbatch_cmd.append(job_file) - result = subprocess.run(sbatch_cmd, capture_output=True) - job_id = int(result.stdout) - print(f' └── Submitted batch job {job_id}') - - if not job_name in self.job_ids['current']: - self.job_ids['current'][job_name] = [job_id] - else: - self.job_ids['current'][job_name].append(job_id) - - return result, job_id + return job_file def wait_for_previous(self): """wait for all jobs of the previous stage to be finished diff --git a/jobs/icon.py b/jobs/icon.py index 441628af..e6a5f8f9 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -66,17 +66,18 @@ def main(cfg): script.write_text(script_str) # Submit run script - result, job_id = cfg.submit('icon', script) + job_id = cfg.submit('icon', script) + # TODO: check # Anything hapenning after submission only makes sense in sequential mode - if not cfg.is_async: - exitcode = result.returncode - - # In case of ICON-ART, ignore the "invalid pointer" error on successful run - if cfg.workflow_name.startswith('icon-art'): - if tools.grep("free(): invalid pointer", logfile)['success'] and \ - tools.grep("clean-up finished", logfile)['success']: - exitcode = 0 - - if exitcode != 0: - raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + #if not cfg.is_async: + # exitcode = result.returncode + # + # # In case of ICON-ART, ignore the "invalid pointer" error on successful run + # if cfg.workflow_name.startswith('icon-art'): + # if tools.grep("free(): invalid pointer", logfile)['success'] and \ + # tools.grep("clean-up finished", logfile)['success']: + # exitcode = 0 + # + # if exitcode != 0: + # raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index aeaf9812..2dc6ad33 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -134,7 +134,7 @@ def main(cfg): with (script := cfg.icon_work / 'copy_input.job').open('w') as f: f.write('\n'.join(script_lines)) - _, copy_id = cfg.submit('prepare_data', script) + copy_id = cfg.submit('prepare_data', script) if cfg.workflow_name == 'icon-art-global': async_error(cfg, part='global ICON-ART') diff --git a/run_chain.py b/run_chain.py index 012fabc3..bcdc5ce5 100755 --- a/run_chain.py +++ b/run_chain.py @@ -230,13 +230,15 @@ def run_chunk(cfg, force, resume): # Submit the job #getattr(jobs, job).main(cfg) - result, job_id = cfg.submit_job_as_sbatch(job, logfile) + script = cfg.create_sbatch_script(job, logfile) + job_id = cfg.submit(job, script) # Logging job_end_time = datetime.now() job_duration = job_end_time - job_launch_time cfg.log_job_status(job, 'FINISH', job_end_time, job_duration) - shutil.copy(logfile, logfile_finish) + # This needs to be done by the job + #shutil.copy(logfile, logfile_finish) # wait for previous chunk to be done cfg.wait_for_previous() From bb8844f252d7d07335b7b3fa29030087a1069fb3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 4 Jan 2024 10:33:00 +0000 Subject: [PATCH 056/245] GitHub Action: Apply Pep8-formatting --- config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index c38d5b19..5efd8a8c 100644 --- a/config.py +++ b/config.py @@ -470,7 +470,8 @@ def create_sbatch_script(self, job_name, log_file): f'#SBATCH --open-mode=append', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', - f'#SBATCH --constraint={self.constraint}', '', + f'#SBATCH --constraint={self.constraint}', + '', f'cd {self.chain_src_dir}', 'eval "$(conda shell.bash hook)"', 'conda activate proc-chain', From 0b5faf16fb58bcd716cf6c952de9d6d8cc51cb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 11:33:53 +0100 Subject: [PATCH 057/245] Fix prepare_data --- jobs/prepare_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 2dc6ad33..b1a5e7c1 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -347,7 +347,7 @@ def main(cfg): dependencies = (copy_id, icontools_id) else: dependencies = copy_id - _, icontools_id = cfg.submit(runscript, + icontools_id = cfg.submit(runscript, runscript_path, add_dep=dependencies) From e64cd487c27e0fd9f447c183ee068edd9872064b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 13:12:53 +0100 Subject: [PATCH 058/245] Disable logging for individual jobs --- config.py | 12 ++++-------- run_chain.py | 25 ++++++++++++++++++------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/config.py b/config.py index 5efd8a8c..875d4970 100644 --- a/config.py +++ b/config.py @@ -464,7 +464,7 @@ def submit(self, job_name, script, add_dep=None): def create_sbatch_script(self, job_name, log_file): script_lines = [ '#!/usr/bin/env bash', - f'#SBATCH --job-name="{job_name}"', + f'#SBATCH --job-name="{job_name}_{self.job_id}"', f'#SBATCH --nodes=1', f'#SBATCH --output={log_file}', f'#SBATCH --open-mode=append', @@ -475,7 +475,7 @@ def create_sbatch_script(self, job_name, log_file): f'cd {self.chain_src_dir}', 'eval "$(conda shell.bash hook)"', 'conda activate proc-chain', - f'./run_chain.py {self.casename} -j {job_name} -c {self.job_id} -f -s', + f'./run_chain.py {self.casename} -j {job_name} -c {self.job_id} -f -s --no-logging', '', ] @@ -496,8 +496,8 @@ def wait_for_previous(self): for ids in self.job_ids['previous'].values(): dep_ids.extend(ids) if dep_ids: - job_file = self.chain_root / 'submit.wait.slurm' - log_file = self.chain_root / 'wait.log' + job_file = self.case_root / 'submit.wait.slurm' + log_file = self.case_root / 'wait.log' dep_str = ':'.join(map(str, dep_ids)) script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="wait"', @@ -513,10 +513,6 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) - # Remove sbatch script and log file after execution - os.remove(job_file) - os.remove(log_file) - class InvalidWorkflowType(Exception): pass diff --git a/run_chain.py b/run_chain.py index c8d768b7..d8aeed5f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -70,6 +70,13 @@ def parse_arguments(): action='store_true', help=sync_help) + no_logging_help = ("Disable logging for chain_status.log.") + parser.add_argument("--no-logging", + action='store_false', + dest="enable_logging", + default=True, + help=no_logging_help) + force_help = ("Force the processing chain to redo all specified jobs," " even if they have been started already or were finished" " previously. WARNING: Only logfiles get deleted," @@ -232,7 +239,6 @@ def run_chunk(cfg, force, resume): cfg.log_job_status(job, 'START', job_launch_time) # Submit the job - #getattr(jobs, job).main(cfg) script = cfg.create_sbatch_script(job, logfile) job_id = cfg.submit(job, script) @@ -451,6 +457,9 @@ def main(): # Make ntry a Config variable cfg.ntry = args.ntry + # Check logging settings + cfg.logging = args.enable_logging + # Convert relative to absolute paths cfg.convert_paths_to_absolute() @@ -481,13 +490,14 @@ def main(): # Print config before chain starts cfg.print_config() + tools.create_dir(cfg.case_root, "case_root") print( f"Starting chain for case {casename} and workflow {cfg.workflow_name}" ) - launch_time = datetime.now() - tools.create_dir(cfg.case_root, "case_root") - cfg.log_job_status('chain', 'START', launch_time) + if cfg.logging: + launch_time = datetime.now() + cfg.log_job_status('chain', 'START', launch_time) # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: @@ -505,9 +515,10 @@ def main(): cfg.enddate_sim = cfg.enddate run_chunk(cfg=cfg, force=args.force, resume=args.resume) - end_time = datetime.now() - duration = end_time - launch_time - cfg.log_job_status('chain', 'FINISH', end_time, duration) + if cfg.logging: + end_time = datetime.now() + duration = end_time - launch_time + cfg.log_job_status('chain', 'FINISH', end_time, duration) print('>>> Finished the processing chain successfully <<<') From efd94d60c0598086d5c1877742750deb1fcec58c Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 4 Jan 2024 12:13:22 +0000 Subject: [PATCH 059/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index b1a5e7c1..52fa51b7 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -348,8 +348,8 @@ def main(cfg): else: dependencies = copy_id icontools_id = cfg.submit(runscript, - runscript_path, - add_dep=dependencies) + runscript_path, + add_dep=dependencies) # If COSMO (and not ICON): else: From 6871eff93d87d4e66683fef9c249015c1f240df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 13:15:15 +0100 Subject: [PATCH 060/245] Write casename into logfile --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 875d4970..35640ef5 100644 --- a/config.py +++ b/config.py @@ -347,7 +347,7 @@ def log_job_status(self, job, status, launch_time, duration=None): """ log_file = self.case_root / "chain_status.log" - # Check if the header exists, if not, create it + # Check if the file exists, if not, create it and write header if not log_file.is_file(): header = "Name ID Status Time Duration\n" with open(log_file, 'w') as f: @@ -357,7 +357,7 @@ def log_job_status(self, job, status, launch_time, duration=None): if job == 'chain': if duration is not None: duration = self.format_duration(duration) - job_id = '' + job_id = self.casename else: if duration is not None: duration = f"{str(int(duration.total_seconds()))} s" From cee8c44bb22c7c427a90e9ec0393b80d507b9f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 14:00:55 +0100 Subject: [PATCH 061/245] Fix timers for jobs --- config.py | 15 ++++++++++++++- jobs/geosp.py | 3 +++ jobs/icon.py | 3 +++ jobs/prepare_data.py | 3 +++ run_chain.py | 17 +++-------------- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/config.py b/config.py index 35640ef5..1e98fcba 100644 --- a/config.py +++ b/config.py @@ -2,8 +2,10 @@ import os import yaml import logging + from jobs import tools -from pathlib import Path +from pathlib import Path +from datetime import datetime class Config(): @@ -391,6 +393,17 @@ def format_duration(self, duration): formatted_duration = f"{int(days)}d {int(hours)}h {int(minutes)}m {int(seconds)}s" return formatted_duration + def init_time_logging(self, job): + launch_time = datetime.now() + self.log_job_status(job, 'START', launch_time) + + return launch_time + + def finish_time_logging(self, job, launch_time): + end_time = datetime.now() + duration = end_time - launch_time + self.log_job_status(job, 'FINISH', end_time, duration) + def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" diff --git a/jobs/geosp.py b/jobs/geosp.py index ba8c93a5..6e243a0f 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -15,6 +15,7 @@ def main(cfg): Add GEOSP """ prepare_data.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("geosp") #----------------------------------------------------- # Add GEOSP to all meteo files @@ -58,3 +59,5 @@ def main(cfg): logging.info("Added GEOSP to file {}".format(merged_file)) # Rename file to get original file name tools.rename_file(merged_file, src_file) + + cfg.finish_time_logging("geosp", launch_time) \ No newline at end of file diff --git a/jobs/icon.py b/jobs/icon.py index e6a5f8f9..02a0240a 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -33,6 +33,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_data.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("icon") logfile = cfg.log_working_dir / "icon" logfile_finish = cfg.log_finished_dir / "icon" @@ -81,3 +82,5 @@ def main(cfg): # # if exitcode != 0: # raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + + cfg.finish_time_logging("icon", launch_time) \ No newline at end of file diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 52fa51b7..7c013dd0 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -102,6 +102,7 @@ def main(cfg): If any subprocess returns a non-zero exit code during execution. """ set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("prepare_data") if cfg.workflow_name.startswith('icon'): logging.info('ICON input data (IC/BC)') @@ -525,3 +526,5 @@ def main(cfg): output_log=True) logging.info("OK") + + cfg.finish_time_logging("prepare_data", launch_time) \ No newline at end of file diff --git a/run_chain.py b/run_chain.py index d8aeed5f..b66f3770 100755 --- a/run_chain.py +++ b/run_chain.py @@ -235,20 +235,11 @@ def run_chunk(cfg, force, resume): logfile = cfg.log_working_dir / job logfile_finish = cfg.log_finished_dir / job tools.change_logfile(logfile) - job_launch_time = datetime.now() - cfg.log_job_status(job, 'START', job_launch_time) # Submit the job script = cfg.create_sbatch_script(job, logfile) job_id = cfg.submit(job, script) - # Logging - job_end_time = datetime.now() - job_duration = job_end_time - job_launch_time - cfg.log_job_status(job, 'FINISH', job_end_time, job_duration) - # This needs to be done by the job - #shutil.copy(logfile, logfile_finish) - # wait for previous chunk to be done cfg.wait_for_previous() # cycle @@ -496,8 +487,7 @@ def main(): ) if cfg.logging: - launch_time = datetime.now() - cfg.log_job_status('chain', 'START', launch_time) + launch_time = cfg.init_time_logging('chain') # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: @@ -516,9 +506,8 @@ def main(): run_chunk(cfg=cfg, force=args.force, resume=args.resume) if cfg.logging: - end_time = datetime.now() - duration = end_time - launch_time - cfg.log_job_status('chain', 'FINISH', end_time, duration) + cfg.finish_time_logging('chain', launch_time) + print('>>> Finished the processing chain successfully <<<') From f60cd9da95ee41b3ce981fdd87f2b053b79d1338 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 4 Jan 2024 13:01:24 +0000 Subject: [PATCH 062/245] GitHub Action: Apply Pep8-formatting --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 1e98fcba..6b6faaa7 100644 --- a/config.py +++ b/config.py @@ -4,7 +4,7 @@ import logging from jobs import tools -from pathlib import Path +from pathlib import Path from datetime import datetime From aefab40dad337621ca6f7c9181532e54df3db905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 4 Jan 2024 16:40:01 +0100 Subject: [PATCH 063/245] Use UTC times --- run_chain.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index b66f3770..8d3dbcbb 100755 --- a/run_chain.py +++ b/run_chain.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from datetime import datetime, timedelta +import pytz import logging import os @@ -352,9 +353,9 @@ def restart_runs(cfg, force, resume): cfg.startdate_sim_yyyymmddhh = job_id[0:10] cfg.enddate_sim_yyyymmddhh = job_id[-10:] cfg.startdate_sim = datetime.strptime(cfg.startdate_sim_yyyymmddhh, - "%Y%m%d%H") + "%Y%m%d%H").replace(tzinfo=pytz.UTC) cfg.enddate_sim = datetime.strptime(cfg.enddate_sim_yyyymmddhh, - "%Y%m%d%H") + "%Y%m%d%H").replace(tzinfo=pytz.UTC) # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." From 23cfddf42f0e0d88f940983d514e76498d8d1e91 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 4 Jan 2024 15:40:35 +0000 Subject: [PATCH 064/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run_chain.py b/run_chain.py index 8d3dbcbb..1601cb63 100755 --- a/run_chain.py +++ b/run_chain.py @@ -352,10 +352,10 @@ def restart_runs(cfg, force, resume): cfg.job_id = job_id cfg.startdate_sim_yyyymmddhh = job_id[0:10] cfg.enddate_sim_yyyymmddhh = job_id[-10:] - cfg.startdate_sim = datetime.strptime(cfg.startdate_sim_yyyymmddhh, - "%Y%m%d%H").replace(tzinfo=pytz.UTC) - cfg.enddate_sim = datetime.strptime(cfg.enddate_sim_yyyymmddhh, - "%Y%m%d%H").replace(tzinfo=pytz.UTC) + cfg.startdate_sim = datetime.strptime( + cfg.startdate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) + cfg.enddate_sim = datetime.strptime( + cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." From f61b8d5a82738b0657dca4480de15a3650cbf42d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 5 Jan 2024 13:02:55 +0100 Subject: [PATCH 065/245] More async jobs and workflows --- .../config.yaml | 4 +- .../icon_runjob.cfg | 0 .../icontools_remap_00_lbc_runjob.cfg | 0 .../icontools_remap_ic_runjob.cfg | 0 .../icontools_remap_lbc_rest_runjob.cfg | 0 cases/icon-test/config.yaml | 2 +- config.py | 14 +- jobs/__init__.py | 3 + jobs/geosp.py | 5 +- jobs/icon.py | 15 -- jobs/prepare_art-oem.py | 173 ------------ jobs/prepare_art.py | 129 +-------- jobs/prepare_art_global.py | 248 ++++++++++++++++++ jobs/prepare_art_oem.py | 109 ++++++++ run_chain.py | 4 +- workflows.yaml | 109 ++++++-- 16 files changed, 466 insertions(+), 349 deletions(-) rename cases/{icon-async-test => icon-seq-test}/config.yaml (94%) rename cases/{icon-async-test => icon-seq-test}/icon_runjob.cfg (100%) rename cases/{icon-async-test => icon-seq-test}/icontools_remap_00_lbc_runjob.cfg (100%) rename cases/{icon-async-test => icon-seq-test}/icontools_remap_ic_runjob.cfg (100%) rename cases/{icon-async-test => icon-seq-test}/icontools_remap_lbc_rest_runjob.cfg (100%) delete mode 100644 jobs/prepare_art-oem.py create mode 100644 jobs/prepare_art_global.py create mode 100644 jobs/prepare_art_oem.py diff --git a/cases/icon-async-test/config.yaml b/cases/icon-seq-test/config.yaml similarity index 94% rename from cases/icon-async-test/config.yaml rename to cases/icon-seq-test/config.yaml index 1000bebf..cee58297 100644 --- a/cases/icon-async-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -1,6 +1,6 @@ -# Configuration file for the 'icon-async-test' case with ICON +# Configuration file for the 'icon-test' case with ICON -workflow: icon-async +workflow: icon-seq constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-async-test/icon_runjob.cfg b/cases/icon-seq-test/icon_runjob.cfg similarity index 100% rename from cases/icon-async-test/icon_runjob.cfg rename to cases/icon-seq-test/icon_runjob.cfg diff --git a/cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg similarity index 100% rename from cases/icon-async-test/icontools_remap_00_lbc_runjob.cfg rename to cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg diff --git a/cases/icon-async-test/icontools_remap_ic_runjob.cfg b/cases/icon-seq-test/icontools_remap_ic_runjob.cfg similarity index 100% rename from cases/icon-async-test/icontools_remap_ic_runjob.cfg rename to cases/icon-seq-test/icontools_remap_ic_runjob.cfg diff --git a/cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg similarity index 100% rename from cases/icon-async-test/icontools_remap_lbc_rest_runjob.cfg rename to cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index c4f79d0c..261a0871 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -1,4 +1,4 @@ -# Configuration file for the 'icon-test' case with ICON +# Configuration file for the 'icon-async-test' case with ICON workflow: icon constraint: gpu diff --git a/config.py b/config.py index 6b6faaa7..c3d3629f 100644 --- a/config.py +++ b/config.py @@ -448,7 +448,7 @@ def get_dep_cmd(self, job_name, add_dep=None): # sequential case return '--wait' - def submit(self, job_name, script, add_dep=None): + def submit(self, job_name, script, add_dep=None, logfile=None): """Submit job with dependencies""" script_path = Path(script) @@ -469,10 +469,20 @@ def submit(self, job_name, script, add_dep=None): self.job_ids['current'][job_name].append(job_id) exitcode = result.returncode + self.check_job(exitcode, logfile) + + return job_id + + def check_job(self, exitcode, logfile=None): + # In case of ICON-ART, ignore the "invalid pointer" error on successful run + if logfile and tools.grep("ART: ", logfile)['success'] and \ + tools.grep("free(): invalid pointer", logfile)['success'] and \ + tools.grep("clean-up finished", logfile)['success']: + exitcode = 0 + if exitcode != 0: raise RuntimeError(f"sbatch returned exitcode {exitcode}") - return job_id def create_sbatch_script(self, job_name, log_file): script_lines = [ diff --git a/jobs/__init__.py b/jobs/__init__.py index b74b46f3..094d28c7 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -15,6 +15,9 @@ from . import photo_rate from . import post_cosmo from . import post_int2lm +from . import prepare_art +from . import prepare_art_oem +from . import prepare_art_global from . import prepare_data from . import reduce_output from . import verify_chain diff --git a/jobs/geosp.py b/jobs/geosp.py index 6e243a0f..25decf3c 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -12,14 +12,11 @@ def main(cfg): """ - Add GEOSP + Add GEOSP to all meteo files """ prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("geosp") - #----------------------------------------------------- - # Add GEOSP to all meteo files - #----------------------------------------------------- for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, cfg.meteo['inc']): # Specify file names diff --git a/jobs/icon.py b/jobs/icon.py index 02a0240a..617bc6f6 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import logging -import subprocess from . import tools, prepare_data @@ -69,18 +68,4 @@ def main(cfg): # Submit run script job_id = cfg.submit('icon', script) - # TODO: check - # Anything hapenning after submission only makes sense in sequential mode - #if not cfg.is_async: - # exitcode = result.returncode - # - # # In case of ICON-ART, ignore the "invalid pointer" error on successful run - # if cfg.workflow_name.startswith('icon-art'): - # if tools.grep("free(): invalid pointer", logfile)['success'] and \ - # tools.grep("clean-up finished", logfile)['success']: - # exitcode = 0 - # - # if exitcode != 0: - # raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) - cfg.finish_time_logging("icon", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art-oem.py b/jobs/prepare_art-oem.py deleted file mode 100644 index f027db36..00000000 --- a/jobs/prepare_art-oem.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -from pathlib import Path -import logging -import shutil -import subprocess -from datetime import timedelta -import xarray as xr -import numpy as np -from . import tools, prepare_data -from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond -from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -from calendar import monthrange - - -def main(cfg): - """ - Add GEOSP - """ - prepare_data.set_cfg_variables(cfg) - - #----------------------------------------------------- - # Add GEOSP to all meteo files - #----------------------------------------------------- - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - # Specify file names - geosp_filename = time.replace( - hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) - - # Copy GEOSP file from last run if not present - if not os.path.exists(geosp_file): - geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) - tools.copy_file(geosp_src_file, - cfg.icon_input_icbc, - output_log=True) - - # Load GEOSP data array as da_geosp at time 00: - ds = xr.open_dataset(src_file) - ds_geosp = xr.open_dataset(geosp_file) - da_geosp = ds_geosp['GEOSP'] - - # Merge GEOSP-dataset with other timesteps - if (time.hour != 0): - # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) - # Merge GEOSP into temporary file - ds_merged = xr.merge([ds, da_geosp]) - ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Logging info for merging GEOSP - logging.info("Added GEOSP to file {}".format(merged_file)) - # Rename file to get original file name - tools.rename_file(merged_file, src_file) - - #----------------------------------------------------- - # Add Q (copy of QV) and/or PS to initial file - #----------------------------------------------------- - if cfg.workflow_name.startswith('icon-art'): - meteo_file = os.path.join( - cfg.icon_input_icbc, - cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') - if os.path.isfile(meteo_file): - merged_file = os.path.join( - cfg.icon_input_icbc, - cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') - ds = xr.open_dataset(meteo_file) - merging = False - if 'PS' not in ds: - if 'LNPS' not in ds: - raise KeyError( - f"'LNPS' must be found in the initial conditions file {meteo_file}" - ) - merging = True - ds['PS'] = ds['LNPS'] - ds['PS'].attrs = ds['LNPS'].attrs - ds['PS'] = np.exp(ds['PS']) - ds['PS'] = ds['PS'].squeeze(dim='lev_2') - ds['PS'].attrs["long_name"] = 'surface pressure' - ds['PS'].attrs['units'] = 'Pa' - logging.info(f"Added PS to file {meteo_file}") - if 'Q' not in ds: - merging = True - ds['Q'] = ds['QV'] - logging.info(f"Added Q to file {meteo_file}") - if merging: - ds.to_netcdf(merged_file) - tools.rename_file(merged_file, meteo_file) - - #----------------------------------------------------- - # In case of OEM: merge chem tracers with meteo-files - #----------------------------------------------------- - if cfg.workflow_name == 'icon-art-oem': - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - if time == cfg.startdate_sim: - #------------ - # Merge IC: - #------------ - meteo_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') - if os.path.isfile(meteo_file): - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem['nameformat']) + '.nc') - merged_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc') - ds_meteo = xr.open_dataset(meteo_file) - ds_chem = xr.open_dataset(chem_file) - # LNPS --> PS - ds_chem['PS'] = ds_chem['LNPS'] - ds_chem['PS'].attrs = ds_chem['LNPS'].attrs - ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') - ds_chem['PS'].attrs["long_name"] = 'surface pressure' - # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") - #ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Rename file to get original file name - tools.rename_file(merged_file, meteo_file) - tools.remove_file(chem_file) - logging.info( - "Added chemical tracer to file {}".format(merged_file)) - - #------------ - # Merge LBC: - #------------ - meteo_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + - '_lbc.nc') - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem_nameformat) + '_lbc.nc') - merged_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + - '_merged.nc') - ds_meteo = xr.open_dataset(meteo_file) - ds_chem = xr.open_dataset(chem_file) - # LNPS --> PS - ds_chem['PS'] = ds_chem['LNPS'] - ds_chem['PS'].attrs = ds_chem['LNPS'].attrs - ds_chem['PS'].attrs["long_name"] = 'surface pressure' - ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] - # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") - #ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Rename file to get original file name - tools.rename_file(merged_file, meteo_file) - tools.remove_file(chem_file) - logging.info( - "Added chemical tracer to file {}".format(merged_file)) diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index f027db36..fbd01770 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -2,72 +2,20 @@ # -*- coding: utf-8 -*- import os -from pathlib import Path import logging -import shutil -import subprocess -from datetime import timedelta import xarray as xr import numpy as np from . import tools, prepare_data -from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond -from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -from calendar import monthrange def main(cfg): """ - Add GEOSP + Add Q (copy of QV) and/or PS to initial file """ prepare_data.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("prepare_art") - #----------------------------------------------------- - # Add GEOSP to all meteo files - #----------------------------------------------------- - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - # Specify file names - geosp_filename = time.replace( - hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) - - # Copy GEOSP file from last run if not present - if not os.path.exists(geosp_file): - geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) - tools.copy_file(geosp_src_file, - cfg.icon_input_icbc, - output_log=True) - - # Load GEOSP data array as da_geosp at time 00: - ds = xr.open_dataset(src_file) - ds_geosp = xr.open_dataset(geosp_file) - da_geosp = ds_geosp['GEOSP'] - - # Merge GEOSP-dataset with other timesteps - if (time.hour != 0): - # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) - # Merge GEOSP into temporary file - ds_merged = xr.merge([ds, da_geosp]) - ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Logging info for merging GEOSP - logging.info("Added GEOSP to file {}".format(merged_file)) - # Rename file to get original file name - tools.rename_file(merged_file, src_file) - - #----------------------------------------------------- - # Add Q (copy of QV) and/or PS to initial file - #----------------------------------------------------- - if cfg.workflow_name.startswith('icon-art'): + if cfg.art_input_folder: meteo_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + @@ -101,73 +49,4 @@ def main(cfg): ds.to_netcdf(merged_file) tools.rename_file(merged_file, meteo_file) - #----------------------------------------------------- - # In case of OEM: merge chem tracers with meteo-files - #----------------------------------------------------- - if cfg.workflow_name == 'icon-art-oem': - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - if time == cfg.startdate_sim: - #------------ - # Merge IC: - #------------ - meteo_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') - if os.path.isfile(meteo_file): - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem['nameformat']) + '.nc') - merged_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc') - ds_meteo = xr.open_dataset(meteo_file) - ds_chem = xr.open_dataset(chem_file) - # LNPS --> PS - ds_chem['PS'] = ds_chem['LNPS'] - ds_chem['PS'].attrs = ds_chem['LNPS'].attrs - ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') - ds_chem['PS'].attrs["long_name"] = 'surface pressure' - # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], - compat="override") - #ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Rename file to get original file name - tools.rename_file(merged_file, meteo_file) - tools.remove_file(chem_file) - logging.info( - "Added chemical tracer to file {}".format(merged_file)) - - #------------ - # Merge LBC: - #------------ - meteo_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + - '_lbc.nc') - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem_nameformat) + '_lbc.nc') - merged_file = os.path.join( - cfg.icon_input_icbc, - time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + - '_merged.nc') - ds_meteo = xr.open_dataset(meteo_file) - ds_chem = xr.open_dataset(chem_file) - # LNPS --> PS - ds_chem['PS'] = ds_chem['LNPS'] - ds_chem['PS'].attrs = ds_chem['LNPS'].attrs - ds_chem['PS'].attrs["long_name"] = 'surface pressure' - ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] - # merge: - ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") - #ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Rename file to get original file name - tools.rename_file(merged_file, meteo_file) - tools.remove_file(chem_file) - logging.info( - "Added chemical tracer to file {}".format(merged_file)) + cfg.finish_time_logging("prepare_art", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py new file mode 100644 index 00000000..46e5d2a7 --- /dev/null +++ b/jobs/prepare_art_global.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import logging +import xarray as xr +from . import tools +from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond +from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging +from calendar import monthrange + + +def set_cfg_variables(cfg): + if cfg.workflow_name.startswith('cosmo'): + cfg.int2lm_root = cfg.chain_root / 'int2lm' + cfg.int2lm_input = cfg.int2lm_root / 'input' + elif cfg.workflow_name.startswith('icon'): + cfg.icon_base = cfg.chain_root / 'icon' + cfg.icon_input = cfg.icon_base / 'input' + cfg.icon_input_icbc = cfg.icon_input / 'icbc' + cfg.icon_work = cfg.icon_base / 'run' + cfg.icon_output = cfg.icon_base / 'output' + cfg.icon_output_reduced = cfg.icon_base / 'output_reduced' + cfg.icon_restart_out = cfg.icon_base / 'restart' + cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' + cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' + + cfg.input_files_scratch = {} + for dsc, file in cfg.input_files.items(): + cfg.input_files[dsc] = (p := Path(file)) + cfg.input_files_scratch[dsc] = cfg.icon_input / p.name + + cfg.create_vars_from_dicts() + + cfg.ini_datetime_string = cfg.startdate.strftime('%Y-%m-%dT%H:00:00Z') + cfg.end_datetime_string = cfg.enddate.strftime('%Y-%m-%dT%H:00:00Z') + + if cfg.workflow_name == 'icon-art-oem': + cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime( + '%Y%m%d_%H') + + if cfg.workflow_name == 'icon-art-global': + # Nudge type (global or nothing) + cfg.nudge_type = 2 if cfg.era5_global_nudging else 0 + # Time step for global nudging in seconds + cfg.nudging_step_seconds = cfg.nudging_step * 3600 + # Prescribed initial conditions for CH4, CO and/or OH + cfg.iart_init_gas = 4 if cfg.species_inicond else 0 + + if cfg.lrestart == '.TRUE.': + cfg.restart_filename = 'restart_atm_DOM01.nc' + cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename + cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename + + cfg.job_ids['current']['prepare_data'] = [] + + +def main(cfg): + """ + This function prepares the data for global ICON-ART simulations + by downloading necessary meteorological and chemical data, + creating the inicond file, and handling global nudging + conditions. The workflow includes fetching ERA5 data, processing it, + and creating initial conditions for tracers (such as CH4, CO, and OH). + + The main steps performed by this function include: + 1. Downloading ERA5 data and creating the inicond file if specified in the + configuration. + 2. Creating initial conditions for tracers (CH4, CO, OH) if needed, + including handling restart scenarios. + 3. Handling global nudging conditions by downloading and processing ERA5 and + CAMS data if configured. + + Note + ---- + - The function utilizes external scripts for processing ERA5 data + (`icon_era5_inicond.sh`, `icon_era5_nudging.sh`) and CAMS data + (`icon_cams_nudging.sh`). + - The `tools` module provides various utility functions used in the workflow. + + Parameters + ---------- + cfg : Config + Object holding all user-configuration parameters as attributes. + """ + set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("prepare_art_global") + + # -- Download ERA5 data and create the inicond file + if cfg.era5_inicond and cfg.lrestart == '.FALSE.': + # -- Fetch ERA5 data + fetch_era5(cfg.startdate_sim, cfg.icon_input_icbc) + + # -- Copy ERA5 processing script (icon_era5_inicond.job) in workdir + with open(cfg.icon_era5_inijob) as input_file: + to_write = input_file.read() + output_file = os.path.join(cfg.icon_input_icbc, + 'icon_era5_inicond.sh') + with open(output_file, "w") as outf: + outf.write(to_write.format(cfg=cfg)) + + # -- Copy mypartab in workdir + shutil.copy( + os.path.join(os.path.dirname(cfg.icon_era5_inijob), + 'mypartab'), + os.path.join(cfg.icon_input_icbc, 'mypartab')) + + # -- Run ERA5 processing script + process = subprocess.Popen([ + "bash", + os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') + ], + stdout=subprocess.PIPE) + process.communicate() + + # ----------------------------------------------------- + # Create tracer initial conditions + # ----------------------------------------------------- + + # -- Download and add CAMS data to the inicond file if needed + if cfg.species_inicond: + + if cfg.lrestart == '.FALSE.': + + ext_restart = '' + filename = cfg.input_files_scratch_inicond_filename + + # -- Copy the script for processing external tracer data in workdir + with open( + os.path.join( + cfg.case_path, + cfg.icon_species_inijob)) as input_file: + to_write = input_file.read() + output_file = os.path.join(cfg.icon_input_icbc, + cfg.icon_species_inijob) + with open(output_file, "w") as outf: + outf.write( + to_write.format(cfg=cfg, + filename=filename, + ext_restart=ext_restart, + year=cfg.startdate_sim.year, + month=cfg.startdate_sim.month, + day=cfg.startdate_sim.day)) + + # -- Run ERA5 processing script + process = subprocess.Popen(["bash", output_file], + stdout=subprocess.PIPE) + process.communicate() + + # -- Create initial conditions for OH concentrations + if 'TROH' in cfg.species2restart: + create_oh_for_inicond(cfg, cfg.startdate_sim.month) + + else: + + # -- Check the extension of tracer variables in the restart file + ds_restart = xr.open_dataset(cfg.restart_file) + tracer_name = cfg.species2restart[0] + # FIXME: + # var_restart = [ + # IndexError: list index out of range + # var_restart = [ + # var for var in ds_restart.data_vars.keys() + # if var.startswith(tracer_name) + # ][0] + # ext_restart = var_restart.replace(tracer_name, '') + + # -- Change OH concentrations in the restart file + # if 'TROH' in cfg.species2restart: + # create_oh_for_restart(cfg, cfg.startdate_sim.month, + # ext_restart) + + # ----------------------------------------------------- + # Create meteorological and tracer nudging conditions + # ----------------------------------------------------- + + # -- If global nudging, download and process ERA5 and CAMS data + if cfg.era5_global_nudging: + + for time in tools.iter_hours(cfg.startdate_sim, + cfg.enddate_sim, + step=cfg.nudging_step): + + # -- Give a name to the nudging file + timestr = time.strftime('%Y%m%d%H') + filename = 'era2icon_R2B03_{timestr}_nudging.nc'.format( + timestr=timestr) + + # -- If initial time, copy the initial conditions to be used as boundary conditions + if time == cfg.startdate_sim and cfg.era5_inicond: + shutil.copy( + cfg.input_files_scratch_inicond_filename, + os.path.join(cfg.icon_input_icbc, filename)) + continue + + # -- Fetch ERA5 data + fetch_era5_nudging(time, cfg.icon_input_icbc) + + # -- Copy ERA5 processing script (icon_era5_nudging.job) in workdir + with open(cfg.icon_era5_nudgingjob) as input_file: + to_write = input_file.read() + output_file = os.path.join( + cfg.icon_input_icbc, + 'icon_era5_nudging_{}.sh'.format(timestr)) + with open(output_file, "w") as outf: + outf.write(to_write.format(cfg=cfg, filename=filename)) + + # -- Copy mypartab in workdir + if not os.path.exists( + os.path.join(cfg.icon_input_icbc, 'mypartab')): + shutil.copy( + os.path.join( + os.path.dirname(cfg.icon_era5_nudgingjob), + 'mypartab'), + os.path.join(cfg.icon_input_icbc, 'mypartab')) + + # -- Run ERA5 processing script + process = subprocess.Popen([ + "bash", + os.path.join(cfg.icon_input_icbc, + 'icon_era5_nudging_{}.sh'.format(timestr)) + ], + stdout=subprocess.PIPE) + process.communicate() + + if cfg.species_global_nudging: + + # -- Copy CAMS processing script (icon_cams_nudging.job) in workdir + with open(cfg.icon_species_nudgingjob) as input_file: + to_write = input_file.read() + output_file = os.path.join( + cfg.icon_input_icbc, + 'icon_cams_nudging_{}.sh'.format(timestr)) + with open(output_file, "w") as outf: + outf.write( + to_write.format(cfg=cfg, filename=filename)) + + # -- Run ERA5 processing script + process = subprocess.Popen([ + "bash", + os.path.join( + cfg.icon_input_icbc, + 'icon_cams_nudging_{}.sh'.format(timestr)) + ], + stdout=subprocess.PIPE) + process.communicate() + + cfg.finish_time_logging("prepare_art_global", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py new file mode 100644 index 00000000..05d53cf6 --- /dev/null +++ b/jobs/prepare_art_oem.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import logging +import xarray as xr +from . import tools, prepare_data + + +def main(cfg): + """ + This function sets up the necessary configuration variables, logs the + initialization time, and merges chemical tracers with meteorological files if + OEM (Online Emission Model) is enabled. The merging process involves + combining initial and lateral boundary condition (IC and LBC) files for the + ICON model. + + **IC Merging (Initial Conditions):** + For the initial time step, it merges the IC files by: + - Combining meteorological and chemical files. + - Adjusting variables such as converting LNPS to PS for surface pressure. + - Saving the merged dataset to a new NetCDF file. + - Renaming and cleaning up the files. + + **LBC Merging (Lateral Boundary Conditions):** + For subsequent time steps, it merges the LBC files by: + - Combining meteorological and chemical files. + - Adjusting variables such as converting LNPS to PS for surface pressure + and renaming specific variables. + - Saving the merged dataset to a new NetCDF file. + - Renaming and cleaning up the files. + + This function utilizes the :func:`prepare_data.set_cfg_variables` + function to set configuration variables, :mod:`xarray` (xr) for handling + NetCDF files, and various utility functions from the :mod:`tools` module for + file operations. + """ + prepare_data.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("prepare_art") + + if cfg.input_files['oem_gridded_emissions_nc']: + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + if time == cfg.startdate_sim: + #------------ + # Merge IC: + #------------ + meteo_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '.nc') + if os.path.isfile(meteo_file): + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem['nameformat']) + '.nc') + merged_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc') + ds_meteo = xr.open_dataset(meteo_file) + ds_chem = xr.open_dataset(chem_file) + # LNPS --> PS + ds_chem['PS'] = ds_chem['LNPS'] + ds_chem['PS'].attrs = ds_chem['LNPS'].attrs + ds_chem['PS'] = ds_chem['PS'].squeeze(dim='lev_2') + ds_chem['PS'].attrs["long_name"] = 'surface pressure' + # merge: + ds_merged = xr.merge([ds_meteo, ds_chem], + compat="override") + #ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Rename file to get original file name + tools.rename_file(merged_file, meteo_file) + tools.remove_file(chem_file) + logging.info( + "Added chemical tracer to file {}".format(merged_file)) + + #------------ + # Merge LBC: + #------------ + meteo_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + + '_lbc.nc') + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem_nameformat) + '_lbc.nc') + merged_file = os.path.join( + cfg.icon_input_icbc, + time.strftime(cfg.meteo['prefix'] + cfg.meteo['nameformat']) + + '_merged.nc') + ds_meteo = xr.open_dataset(meteo_file) + ds_chem = xr.open_dataset(chem_file) + # LNPS --> PS + ds_chem['PS'] = ds_chem['LNPS'] + ds_chem['PS'].attrs = ds_chem['LNPS'].attrs + ds_chem['PS'].attrs["long_name"] = 'surface pressure' + ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] + # merge: + ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") + #ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Rename file to get original file name + tools.rename_file(merged_file, meteo_file) + tools.remove_file(chem_file) + logging.info( + "Added chemical tracer to file {}".format(merged_file)) + + cfg.finish_time_logging("prepare_art_oem", launch_time) \ No newline at end of file diff --git a/run_chain.py b/run_chain.py index 1601cb63..dc8fd20e 100755 --- a/run_chain.py +++ b/run_chain.py @@ -255,14 +255,14 @@ def run_chunk(cfg, force, resume): if not force: while True: if (cfg.log_finished_dir / job).exists(): - print(f"Skip {job} for chunk {cfg.job_id}") + print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') skip = True break elif resume: resume = False break else: - print(f"Wait for {job} of chunk {cfg.job_id}") + print(f" └── Wait for {job} of chunk {cfg.job_id}") sys.stdout.flush() for _ in range(3000): time.sleep(0.1) diff --git a/workflows.yaml b/workflows.yaml index a451b072..9c8b8480 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -1,32 +1,91 @@ cosmo: - jobs: ['prepare_data', 'int2lm', 'cosmo', 'post_cosmo'] - features: ['restart', 'spinup'] + jobs: + - prepare_data + - int2lm + - cosmo + - post_cosmo + features: + - restart + - spinup + cosmo-ghg: - jobs: ['prepare_data', 'emissions', 'biofluxes', 'oem', 'online_vprm', - 'int2lm', 'post_int2lm', 'cosmo', 'post_cosmo'] - features: ['restart', 'tracers', 'spinup'] + jobs: + - prepare_data + - emissions + - biofluxes + - oem + - online_vprm + - int2lm + - post_int2lm + - cosmo + - post_cosmo + features: + - restart + - tracers + - spinup + cosmo-art: - jobs: ['prepare_data', 'emissions', 'obs_nudging', 'photo_rate', - 'int2lm', 'cosmo', 'post_cosmo'] - features: ['nesting', 'spinup'] -icon-async: - jobs: ['prepare_data', 'geosp', 'icon'] - features: ['restart'] - dependencies: - geosp: - current: ['prepare_data'] - icon: - current: ['prepare_data', 'geosp'] - previous: ['icon'] + jobs: + - prepare_data + - emissions + - obs_nudging + - photo_rate + - int2lm + - cosmo + - post_cosmo + features: + - nesting + - spinup + icon: - jobs: ['prepare_data', 'icon'] - features: ['restart'] + jobs: + - prepare_data + - geosp + - icon + features: + - restart + dependencies: + geosp: + current: + - prepare_data + icon: + current: + - prepare_data + - geosp + previous: + - icon + +icon-seq: + jobs: + - prepare_data + - geosp + - icon + features: + - restart + icon-art: - jobs: ['prepare_data', 'icon'] - features: ['restart'] + jobs: + - prepare_data + - geosp + - prepare_art + - icon + features: + - restart + icon-art-global: - jobs: ['prepare_data', 'icon'] - features: ['restart'] + jobs: + - prepare_data + - prepare_art_global + - icon + features: + - restart + icon-art-oem: - jobs: ['prepare_data', 'icon'] - features: ['restart'] + jobs: + - prepare_data + - geosp + - prepare_art + - prepare_art_oem + - icon + features: + - restart From 08aeca4b252f92d2188f530d8ddee6c0695358e0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 5 Jan 2024 12:03:36 +0000 Subject: [PATCH 066/245] GitHub Action: Apply Pep8-formatting --- config.py | 3 +-- jobs/prepare_art_global.py | 55 ++++++++++++++++---------------------- run_chain.py | 8 ++++-- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/config.py b/config.py index c3d3629f..481163bf 100644 --- a/config.py +++ b/config.py @@ -479,11 +479,10 @@ def check_job(self, exitcode, logfile=None): tools.grep("free(): invalid pointer", logfile)['success'] and \ tools.grep("clean-up finished", logfile)['success']: exitcode = 0 - + if exitcode != 0: raise RuntimeError(f"sbatch returned exitcode {exitcode}") - def create_sbatch_script(self, job_name, log_file): script_lines = [ '#!/usr/bin/env bash', diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 46e5d2a7..3b0f8b31 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -94,15 +94,13 @@ def main(cfg): # -- Copy ERA5 processing script (icon_era5_inicond.job) in workdir with open(cfg.icon_era5_inijob) as input_file: to_write = input_file.read() - output_file = os.path.join(cfg.icon_input_icbc, - 'icon_era5_inicond.sh') + output_file = os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') with open(output_file, "w") as outf: outf.write(to_write.format(cfg=cfg)) # -- Copy mypartab in workdir shutil.copy( - os.path.join(os.path.dirname(cfg.icon_era5_inijob), - 'mypartab'), + os.path.join(os.path.dirname(cfg.icon_era5_inijob), 'mypartab'), os.path.join(cfg.icon_input_icbc, 'mypartab')) # -- Run ERA5 processing script @@ -110,7 +108,7 @@ def main(cfg): "bash", os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() # ----------------------------------------------------- @@ -126,13 +124,11 @@ def main(cfg): filename = cfg.input_files_scratch_inicond_filename # -- Copy the script for processing external tracer data in workdir - with open( - os.path.join( - cfg.case_path, - cfg.icon_species_inijob)) as input_file: + with open(os.path.join(cfg.case_path, + cfg.icon_species_inijob)) as input_file: to_write = input_file.read() output_file = os.path.join(cfg.icon_input_icbc, - cfg.icon_species_inijob) + cfg.icon_species_inijob) with open(output_file, "w") as outf: outf.write( to_write.format(cfg=cfg, @@ -144,7 +140,7 @@ def main(cfg): # -- Run ERA5 processing script process = subprocess.Popen(["bash", output_file], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() # -- Create initial conditions for OH concentrations @@ -178,8 +174,8 @@ def main(cfg): if cfg.era5_global_nudging: for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - step=cfg.nudging_step): + cfg.enddate_sim, + step=cfg.nudging_step): # -- Give a name to the nudging file timestr = time.strftime('%Y%m%d%H') @@ -188,9 +184,8 @@ def main(cfg): # -- If initial time, copy the initial conditions to be used as boundary conditions if time == cfg.startdate_sim and cfg.era5_inicond: - shutil.copy( - cfg.input_files_scratch_inicond_filename, - os.path.join(cfg.icon_input_icbc, filename)) + shutil.copy(cfg.input_files_scratch_inicond_filename, + os.path.join(cfg.icon_input_icbc, filename)) continue # -- Fetch ERA5 data @@ -200,27 +195,25 @@ def main(cfg): with open(cfg.icon_era5_nudgingjob) as input_file: to_write = input_file.read() output_file = os.path.join( - cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) + cfg.icon_input_icbc, 'icon_era5_nudging_{}.sh'.format(timestr)) with open(output_file, "w") as outf: outf.write(to_write.format(cfg=cfg, filename=filename)) # -- Copy mypartab in workdir - if not os.path.exists( - os.path.join(cfg.icon_input_icbc, 'mypartab')): + if not os.path.exists(os.path.join(cfg.icon_input_icbc, + 'mypartab')): shutil.copy( - os.path.join( - os.path.dirname(cfg.icon_era5_nudgingjob), - 'mypartab'), + os.path.join(os.path.dirname(cfg.icon_era5_nudgingjob), + 'mypartab'), os.path.join(cfg.icon_input_icbc, 'mypartab')) # -- Run ERA5 processing script process = subprocess.Popen([ "bash", os.path.join(cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) + 'icon_era5_nudging_{}.sh'.format(timestr)) ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() if cfg.species_global_nudging: @@ -232,17 +225,15 @@ def main(cfg): cfg.icon_input_icbc, 'icon_cams_nudging_{}.sh'.format(timestr)) with open(output_file, "w") as outf: - outf.write( - to_write.format(cfg=cfg, filename=filename)) + outf.write(to_write.format(cfg=cfg, filename=filename)) # -- Run ERA5 processing script process = subprocess.Popen([ "bash", - os.path.join( - cfg.icon_input_icbc, - 'icon_cams_nudging_{}.sh'.format(timestr)) + os.path.join(cfg.icon_input_icbc, + 'icon_cams_nudging_{}.sh'.format(timestr)) ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() - cfg.finish_time_logging("prepare_art_global", launch_time) \ No newline at end of file + cfg.finish_time_logging("prepare_art_global", launch_time) diff --git a/run_chain.py b/run_chain.py index dc8fd20e..dc6054b8 100755 --- a/run_chain.py +++ b/run_chain.py @@ -255,14 +255,18 @@ def run_chunk(cfg, force, resume): if not force: while True: if (cfg.log_finished_dir / job).exists(): - print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') + print( + f' └── Skip "{job}" for chunk "{cfg.job_id}"' + ) skip = True break elif resume: resume = False break else: - print(f" └── Wait for {job} of chunk {cfg.job_id}") + print( + f" └── Wait for {job} of chunk {cfg.job_id}" + ) sys.stdout.flush() for _ in range(3000): time.sleep(0.1) From 5ce30de65e8af7ba14d5de4a357a55aeb4de2f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 5 Jan 2024 16:57:55 +0100 Subject: [PATCH 067/245] Add logging to new jobs --- jobs/geosp.py | 1 + jobs/icon.py | 2 +- jobs/prepare_art.py | 2 ++ jobs/prepare_art_global.py | 57 +++++++++++++++++++++++--------------- jobs/prepare_art_oem.py | 3 +- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/jobs/geosp.py b/jobs/geosp.py index 25decf3c..f648e599 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -16,6 +16,7 @@ def main(cfg): """ prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("geosp") + logging.info("Add GEOSP to all meteo files") for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, cfg.meteo['inc']): diff --git a/jobs/icon.py b/jobs/icon.py index 617bc6f6..a2932b8b 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -66,6 +66,6 @@ def main(cfg): script.write_text(script_str) # Submit run script - job_id = cfg.submit('icon', script) + job_id = cfg.submit('icon', script, logfile=logfile) cfg.finish_time_logging("icon", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index fbd01770..94fae342 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -14,6 +14,7 @@ def main(cfg): """ prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") + logging.info('Add Q (copy of QV) and/or PS to initial file') if cfg.art_input_folder: meteo_file = os.path.join( @@ -49,4 +50,5 @@ def main(cfg): ds.to_netcdf(merged_file) tools.rename_file(merged_file, meteo_file) + logging.info('OK') cfg.finish_time_logging("prepare_art", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 3b0f8b31..c99ec5a1 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -85,6 +85,7 @@ def main(cfg): """ set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art_global") + logging.info("Preprae ICON-ART for global simulations") # -- Download ERA5 data and create the inicond file if cfg.era5_inicond and cfg.lrestart == '.FALSE.': @@ -94,13 +95,15 @@ def main(cfg): # -- Copy ERA5 processing script (icon_era5_inicond.job) in workdir with open(cfg.icon_era5_inijob) as input_file: to_write = input_file.read() - output_file = os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') + output_file = os.path.join(cfg.icon_input_icbc, + 'icon_era5_inicond.sh') with open(output_file, "w") as outf: outf.write(to_write.format(cfg=cfg)) # -- Copy mypartab in workdir shutil.copy( - os.path.join(os.path.dirname(cfg.icon_era5_inijob), 'mypartab'), + os.path.join(os.path.dirname(cfg.icon_era5_inijob), + 'mypartab'), os.path.join(cfg.icon_input_icbc, 'mypartab')) # -- Run ERA5 processing script @@ -108,7 +111,7 @@ def main(cfg): "bash", os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() # ----------------------------------------------------- @@ -124,11 +127,13 @@ def main(cfg): filename = cfg.input_files_scratch_inicond_filename # -- Copy the script for processing external tracer data in workdir - with open(os.path.join(cfg.case_path, - cfg.icon_species_inijob)) as input_file: + with open( + os.path.join( + cfg.case_path, + cfg.icon_species_inijob)) as input_file: to_write = input_file.read() output_file = os.path.join(cfg.icon_input_icbc, - cfg.icon_species_inijob) + cfg.icon_species_inijob) with open(output_file, "w") as outf: outf.write( to_write.format(cfg=cfg, @@ -140,7 +145,7 @@ def main(cfg): # -- Run ERA5 processing script process = subprocess.Popen(["bash", output_file], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() # -- Create initial conditions for OH concentrations @@ -174,8 +179,8 @@ def main(cfg): if cfg.era5_global_nudging: for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - step=cfg.nudging_step): + cfg.enddate_sim, + step=cfg.nudging_step): # -- Give a name to the nudging file timestr = time.strftime('%Y%m%d%H') @@ -184,8 +189,9 @@ def main(cfg): # -- If initial time, copy the initial conditions to be used as boundary conditions if time == cfg.startdate_sim and cfg.era5_inicond: - shutil.copy(cfg.input_files_scratch_inicond_filename, - os.path.join(cfg.icon_input_icbc, filename)) + shutil.copy( + cfg.input_files_scratch_inicond_filename, + os.path.join(cfg.icon_input_icbc, filename)) continue # -- Fetch ERA5 data @@ -195,25 +201,27 @@ def main(cfg): with open(cfg.icon_era5_nudgingjob) as input_file: to_write = input_file.read() output_file = os.path.join( - cfg.icon_input_icbc, 'icon_era5_nudging_{}.sh'.format(timestr)) + cfg.icon_input_icbc, + 'icon_era5_nudging_{}.sh'.format(timestr)) with open(output_file, "w") as outf: outf.write(to_write.format(cfg=cfg, filename=filename)) # -- Copy mypartab in workdir - if not os.path.exists(os.path.join(cfg.icon_input_icbc, - 'mypartab')): + if not os.path.exists( + os.path.join(cfg.icon_input_icbc, 'mypartab')): shutil.copy( - os.path.join(os.path.dirname(cfg.icon_era5_nudgingjob), - 'mypartab'), + os.path.join( + os.path.dirname(cfg.icon_era5_nudgingjob), + 'mypartab'), os.path.join(cfg.icon_input_icbc, 'mypartab')) # -- Run ERA5 processing script process = subprocess.Popen([ "bash", os.path.join(cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) + 'icon_era5_nudging_{}.sh'.format(timestr)) ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() if cfg.species_global_nudging: @@ -225,15 +233,18 @@ def main(cfg): cfg.icon_input_icbc, 'icon_cams_nudging_{}.sh'.format(timestr)) with open(output_file, "w") as outf: - outf.write(to_write.format(cfg=cfg, filename=filename)) + outf.write( + to_write.format(cfg=cfg, filename=filename)) # -- Run ERA5 processing script process = subprocess.Popen([ "bash", - os.path.join(cfg.icon_input_icbc, - 'icon_cams_nudging_{}.sh'.format(timestr)) + os.path.join( + cfg.icon_input_icbc, + 'icon_cams_nudging_{}.sh'.format(timestr)) ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() - cfg.finish_time_logging("prepare_art_global", launch_time) + logging.info("OK") + cfg.finish_time_logging("prepare_art_global", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 05d53cf6..3ed16d19 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -13,7 +13,7 @@ def main(cfg): initialization time, and merges chemical tracers with meteorological files if OEM (Online Emission Model) is enabled. The merging process involves combining initial and lateral boundary condition (IC and LBC) files for the - ICON model. + ICON-ART model. **IC Merging (Initial Conditions):** For the initial time step, it merges the IC files by: @@ -37,6 +37,7 @@ def main(cfg): """ prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") + logging.info('Merging IC and LBC') if cfg.input_files['oem_gridded_emissions_nc']: for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, From dfd80b05df6e55a879e7eac15fd8db12daecc7e5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 5 Jan 2024 15:58:21 +0000 Subject: [PATCH 068/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_art_global.py | 55 ++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index c99ec5a1..37abb53a 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -95,15 +95,13 @@ def main(cfg): # -- Copy ERA5 processing script (icon_era5_inicond.job) in workdir with open(cfg.icon_era5_inijob) as input_file: to_write = input_file.read() - output_file = os.path.join(cfg.icon_input_icbc, - 'icon_era5_inicond.sh') + output_file = os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') with open(output_file, "w") as outf: outf.write(to_write.format(cfg=cfg)) # -- Copy mypartab in workdir shutil.copy( - os.path.join(os.path.dirname(cfg.icon_era5_inijob), - 'mypartab'), + os.path.join(os.path.dirname(cfg.icon_era5_inijob), 'mypartab'), os.path.join(cfg.icon_input_icbc, 'mypartab')) # -- Run ERA5 processing script @@ -111,7 +109,7 @@ def main(cfg): "bash", os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() # ----------------------------------------------------- @@ -127,13 +125,11 @@ def main(cfg): filename = cfg.input_files_scratch_inicond_filename # -- Copy the script for processing external tracer data in workdir - with open( - os.path.join( - cfg.case_path, - cfg.icon_species_inijob)) as input_file: + with open(os.path.join(cfg.case_path, + cfg.icon_species_inijob)) as input_file: to_write = input_file.read() output_file = os.path.join(cfg.icon_input_icbc, - cfg.icon_species_inijob) + cfg.icon_species_inijob) with open(output_file, "w") as outf: outf.write( to_write.format(cfg=cfg, @@ -145,7 +141,7 @@ def main(cfg): # -- Run ERA5 processing script process = subprocess.Popen(["bash", output_file], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() # -- Create initial conditions for OH concentrations @@ -179,8 +175,8 @@ def main(cfg): if cfg.era5_global_nudging: for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - step=cfg.nudging_step): + cfg.enddate_sim, + step=cfg.nudging_step): # -- Give a name to the nudging file timestr = time.strftime('%Y%m%d%H') @@ -189,9 +185,8 @@ def main(cfg): # -- If initial time, copy the initial conditions to be used as boundary conditions if time == cfg.startdate_sim and cfg.era5_inicond: - shutil.copy( - cfg.input_files_scratch_inicond_filename, - os.path.join(cfg.icon_input_icbc, filename)) + shutil.copy(cfg.input_files_scratch_inicond_filename, + os.path.join(cfg.icon_input_icbc, filename)) continue # -- Fetch ERA5 data @@ -201,27 +196,25 @@ def main(cfg): with open(cfg.icon_era5_nudgingjob) as input_file: to_write = input_file.read() output_file = os.path.join( - cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) + cfg.icon_input_icbc, 'icon_era5_nudging_{}.sh'.format(timestr)) with open(output_file, "w") as outf: outf.write(to_write.format(cfg=cfg, filename=filename)) # -- Copy mypartab in workdir - if not os.path.exists( - os.path.join(cfg.icon_input_icbc, 'mypartab')): + if not os.path.exists(os.path.join(cfg.icon_input_icbc, + 'mypartab')): shutil.copy( - os.path.join( - os.path.dirname(cfg.icon_era5_nudgingjob), - 'mypartab'), + os.path.join(os.path.dirname(cfg.icon_era5_nudgingjob), + 'mypartab'), os.path.join(cfg.icon_input_icbc, 'mypartab')) # -- Run ERA5 processing script process = subprocess.Popen([ "bash", os.path.join(cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) + 'icon_era5_nudging_{}.sh'.format(timestr)) ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() if cfg.species_global_nudging: @@ -233,18 +226,16 @@ def main(cfg): cfg.icon_input_icbc, 'icon_cams_nudging_{}.sh'.format(timestr)) with open(output_file, "w") as outf: - outf.write( - to_write.format(cfg=cfg, filename=filename)) + outf.write(to_write.format(cfg=cfg, filename=filename)) # -- Run ERA5 processing script process = subprocess.Popen([ "bash", - os.path.join( - cfg.icon_input_icbc, - 'icon_cams_nudging_{}.sh'.format(timestr)) + os.path.join(cfg.icon_input_icbc, + 'icon_cams_nudging_{}.sh'.format(timestr)) ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE) process.communicate() logging.info("OK") - cfg.finish_time_logging("prepare_art_global", launch_time) \ No newline at end of file + cfg.finish_time_logging("prepare_art_global", launch_time) From 98ca64baa854cd4ee1a477fbb1bd79fab2eb6314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 5 Jan 2024 16:59:58 +0100 Subject: [PATCH 069/245] Remove unused imports --- jobs/geosp.py | 1 - jobs/prepare_art_global.py | 1 - jobs/prepare_data.py | 1 - 3 files changed, 3 deletions(-) diff --git a/jobs/geosp.py b/jobs/geosp.py index f648e599..5d1929ab 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -4,7 +4,6 @@ import os from pathlib import Path import logging -from datetime import timedelta import xarray as xr import numpy as np from . import tools, prepare_data diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index c99ec5a1..f886da0a 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -7,7 +7,6 @@ from . import tools from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -from calendar import monthrange def set_cfg_variables(cfg): diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 7c013dd0..c9c5e079 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -8,7 +8,6 @@ import subprocess from datetime import timedelta import xarray as xr -import numpy as np from . import tools from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging From 321e62964ef23e1f5c544f245649e33f30232aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 5 Jan 2024 17:06:49 +0100 Subject: [PATCH 070/245] Remove global icon-art from prepare_data --- jobs/prepare_data.py | 267 +++++++++---------------------------------- 1 file changed, 53 insertions(+), 214 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index c9c5e079..67e8167c 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -136,220 +136,59 @@ def main(cfg): copy_id = cfg.submit('prepare_data', script) - if cfg.workflow_name == 'icon-art-global': - async_error(cfg, part='global ICON-ART') - # -- Download ERA5 data and create the inicond file - if cfg.era5_inicond and cfg.lrestart == '.FALSE.': - # -- Fetch ERA5 data - fetch_era5(cfg.startdate_sim, cfg.icon_input_icbc) - - # -- Copy ERA5 processing script (icon_era5_inicond.job) in workdir - with open(cfg.icon_era5_inijob) as input_file: - to_write = input_file.read() - output_file = os.path.join(cfg.icon_input_icbc, - 'icon_era5_inicond.sh') - with open(output_file, "w") as outf: - outf.write(to_write.format(cfg=cfg)) - - # -- Copy mypartab in workdir - shutil.copy( - os.path.join(os.path.dirname(cfg.icon_era5_inijob), - 'mypartab'), - os.path.join(cfg.icon_input_icbc, 'mypartab')) - - # -- Run ERA5 processing script - process = subprocess.Popen([ - "bash", - os.path.join(cfg.icon_input_icbc, 'icon_era5_inicond.sh') - ], - stdout=subprocess.PIPE) - process.communicate() - - # ----------------------------------------------------- - # Create tracer initial conditions - # ----------------------------------------------------- - - # -- Download and add CAMS data to the inicond file if needed - if cfg.species_inicond: - - if cfg.lrestart == '.FALSE.': - - ext_restart = '' - filename = cfg.input_files_scratch_inicond_filename - - # -- Copy the script for processing external tracer data in workdir - with open( - os.path.join( - cfg.case_path, - cfg.icon_species_inijob)) as input_file: - to_write = input_file.read() - output_file = os.path.join(cfg.icon_input_icbc, - cfg.icon_species_inijob) - with open(output_file, "w") as outf: - outf.write( - to_write.format(cfg=cfg, - filename=filename, - ext_restart=ext_restart, - year=cfg.startdate_sim.year, - month=cfg.startdate_sim.month, - day=cfg.startdate_sim.day)) - - # -- Run ERA5 processing script - process = subprocess.Popen(["bash", output_file], - stdout=subprocess.PIPE) - process.communicate() - - # -- Create initial conditions for OH concentrations - if 'TROH' in cfg.species2restart: - create_oh_for_inicond(cfg, cfg.startdate_sim.month) - - else: - - # -- Check the extension of tracer variables in the restart file - ds_restart = xr.open_dataset(cfg.restart_file) - tracer_name = cfg.species2restart[0] - # FIXME: - # var_restart = [ - # IndexError: list index out of range - # var_restart = [ - # var for var in ds_restart.data_vars.keys() - # if var.startswith(tracer_name) - # ][0] - # ext_restart = var_restart.replace(tracer_name, '') - - # -- Change OH concentrations in the restart file - # if 'TROH' in cfg.species2restart: - # create_oh_for_restart(cfg, cfg.startdate_sim.month, - # ext_restart) - - # ----------------------------------------------------- - # Create meteorological and tracer nudging conditions - # ----------------------------------------------------- - - # -- If global nudging, download and process ERA5 and CAMS data - if cfg.era5_global_nudging: - - for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, - step=cfg.nudging_step): - - # -- Give a name to the nudging file - timestr = time.strftime('%Y%m%d%H') - filename = 'era2icon_R2B03_{timestr}_nudging.nc'.format( - timestr=timestr) - - # -- If initial time, copy the initial conditions to be used as boundary conditions - if time == cfg.startdate_sim and cfg.era5_inicond: - shutil.copy( - cfg.input_files_scratch_inicond_filename, - os.path.join(cfg.icon_input_icbc, filename)) - continue - - # -- Fetch ERA5 data - fetch_era5_nudging(time, cfg.icon_input_icbc) - - # -- Copy ERA5 processing script (icon_era5_nudging.job) in workdir - with open(cfg.icon_era5_nudgingjob) as input_file: - to_write = input_file.read() - output_file = os.path.join( - cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) - with open(output_file, "w") as outf: - outf.write(to_write.format(cfg=cfg, filename=filename)) - - # -- Copy mypartab in workdir - if not os.path.exists( - os.path.join(cfg.icon_input_icbc, 'mypartab')): - shutil.copy( - os.path.join( - os.path.dirname(cfg.icon_era5_nudgingjob), - 'mypartab'), - os.path.join(cfg.icon_input_icbc, 'mypartab')) - - # -- Run ERA5 processing script - process = subprocess.Popen([ - "bash", - os.path.join(cfg.icon_input_icbc, - 'icon_era5_nudging_{}.sh'.format(timestr)) - ], - stdout=subprocess.PIPE) - process.communicate() - - if cfg.species_global_nudging: - - # -- Copy CAMS processing script (icon_cams_nudging.job) in workdir - with open(cfg.icon_species_nudgingjob) as input_file: - to_write = input_file.read() - output_file = os.path.join( - cfg.icon_input_icbc, - 'icon_cams_nudging_{}.sh'.format(timestr)) - with open(output_file, "w") as outf: - outf.write( - to_write.format(cfg=cfg, filename=filename)) - - # -- Run ERA5 processing script - process = subprocess.Popen([ - "bash", - os.path.join( - cfg.icon_input_icbc, - 'icon_cams_nudging_{}.sh'.format(timestr)) - ], - stdout=subprocess.PIPE) - process.communicate() - - else: # non-global ICON-ART - #----------------------------------------------------- - # Create LBC datafile lists (each at 00 UTC and others) - #----------------------------------------------------- - datafile_list = [] - datafile_list_rest = [] - datafile_list_chem = [] - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - meteo_file = os.path.join( - cfg.icon_input_icbc, cfg.meteo['prefix'] + - time.strftime(cfg.meteo['nameformat'])) - if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem_nameformat)) - datafile_list_chem.append(chem_file + cfg.chem['suffix']) - if meteo_file.endswith('00'): - datafile_list.append(meteo_file + cfg.meteo['suffix']) - else: - datafile_list_rest.append(meteo_file + cfg.meteo['suffix']) - datafile_list = ' '.join([str(v) for v in datafile_list]) - datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) - datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) - - #----------------------------------------------------- - # Write and submit runscripts - #----------------------------------------------------- - icontools_id = None - for runscript in cfg.icontools_runjobs: - with open(os.path.join(cfg.case_path, - runscript)) as input_file: - to_write = input_file.read() - runscript_path = cfg.icon_work / f"{runscript}.job" - with open(runscript_path, "w") as outf: - outf.write( - to_write.format(cfg=cfg, - meteo=cfg.meteo, - logfile=logfile, - logfile_finish=logfile_finish, - datafile_list=datafile_list, - datafile_list_rest=datafile_list_rest, - datafile_list_chem=datafile_list_chem)) - - # Submitting icontools runscripts sequentially - logging.info(f" Starting icontools runscript {runscript}.") - if icontools_id: - dependencies = (copy_id, icontools_id) - else: - dependencies = copy_id - icontools_id = cfg.submit(runscript, - runscript_path, - add_dep=dependencies) + + #----------------------------------------------------- + # Create LBC datafile lists (each at 00 UTC and others) + #----------------------------------------------------- + datafile_list = [] + datafile_list_rest = [] + datafile_list_chem = [] + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + meteo_file = os.path.join( + cfg.icon_input_icbc, cfg.meteo['prefix'] + + time.strftime(cfg.meteo['nameformat'])) + if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': + chem_file = os.path.join( + cfg.icon_input_icbc, cfg.chem['prefix'] + + time.strftime(cfg.chem_nameformat)) + datafile_list_chem.append(chem_file + cfg.chem['suffix']) + if meteo_file.endswith('00'): + datafile_list.append(meteo_file + cfg.meteo['suffix']) + else: + datafile_list_rest.append(meteo_file + cfg.meteo['suffix']) + datafile_list = ' '.join([str(v) for v in datafile_list]) + datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) + datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) + + #----------------------------------------------------- + # Write and submit runscripts + #----------------------------------------------------- + icontools_id = None + for runscript in cfg.icontools_runjobs: + with open(os.path.join(cfg.case_path, + runscript)) as input_file: + to_write = input_file.read() + runscript_path = cfg.icon_work / f"{runscript}.job" + with open(runscript_path, "w") as outf: + outf.write( + to_write.format(cfg=cfg, + meteo=cfg.meteo, + logfile=logfile, + logfile_finish=logfile_finish, + datafile_list=datafile_list, + datafile_list_rest=datafile_list_rest, + datafile_list_chem=datafile_list_chem)) + + # Submitting icontools runscripts sequentially + logging.info(f" Starting icontools runscript {runscript}.") + if icontools_id: + dependencies = (copy_id, icontools_id) + else: + dependencies = copy_id + icontools_id = cfg.submit(runscript, + runscript_path, + add_dep=dependencies) # If COSMO (and not ICON): else: From 8b95db71140ce3ea1f1819d452e38b576a0d815b Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 5 Jan 2024 16:07:12 +0000 Subject: [PATCH 071/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 67e8167c..83e93bed 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -136,7 +136,6 @@ def main(cfg): copy_id = cfg.submit('prepare_data', script) - #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) #----------------------------------------------------- @@ -144,14 +143,14 @@ def main(cfg): datafile_list_rest = [] datafile_list_chem = [] for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): meteo_file = os.path.join( - cfg.icon_input_icbc, cfg.meteo['prefix'] + - time.strftime(cfg.meteo['nameformat'])) + cfg.icon_input_icbc, + cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem_nameformat)) + cfg.icon_input_icbc, + cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) datafile_list_chem.append(chem_file + cfg.chem['suffix']) if meteo_file.endswith('00'): datafile_list.append(meteo_file + cfg.meteo['suffix']) @@ -166,8 +165,7 @@ def main(cfg): #----------------------------------------------------- icontools_id = None for runscript in cfg.icontools_runjobs: - with open(os.path.join(cfg.case_path, - runscript)) as input_file: + with open(os.path.join(cfg.case_path, runscript)) as input_file: to_write = input_file.read() runscript_path = cfg.icon_work / f"{runscript}.job" with open(runscript_path, "w") as outf: @@ -187,8 +185,8 @@ def main(cfg): else: dependencies = copy_id icontools_id = cfg.submit(runscript, - runscript_path, - add_dep=dependencies) + runscript_path, + add_dep=dependencies) # If COSMO (and not ICON): else: @@ -365,4 +363,4 @@ def main(cfg): logging.info("OK") - cfg.finish_time_logging("prepare_data", launch_time) \ No newline at end of file + cfg.finish_time_logging("prepare_data", launch_time) From cdbb6927fc6b90c555e256a8b8ae8bee51f63adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 5 Jan 2024 17:10:18 +0100 Subject: [PATCH 072/245] Replace os.path with Pathlib --- jobs/prepare_data.py | 83 +++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 47 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 67e8167c..b42ab1f0 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -65,6 +65,8 @@ def async_error(cfg, part="This part"): f"{part} isn't ready for async execution yet") +# ... (previous code remains unchanged) + def main(cfg): """ **ICON and COSMO Data Preparation** @@ -145,13 +147,11 @@ def main(cfg): datafile_list_chem = [] for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, cfg.meteo['inc']): - meteo_file = os.path.join( - cfg.icon_input_icbc, cfg.meteo['prefix'] + - time.strftime(cfg.meteo['nameformat'])) + meteo_file = cfg.icon_input_icbc / ( + cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': - chem_file = os.path.join( - cfg.icon_input_icbc, cfg.chem['prefix'] + - time.strftime(cfg.chem_nameformat)) + chem_file = cfg.icon_input_icbc / ( + cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) datafile_list_chem.append(chem_file + cfg.chem['suffix']) if meteo_file.endswith('00'): datafile_list.append(meteo_file + cfg.meteo['suffix']) @@ -166,11 +166,10 @@ def main(cfg): #----------------------------------------------------- icontools_id = None for runscript in cfg.icontools_runjobs: - with open(os.path.join(cfg.case_path, - runscript)) as input_file: + with (cfg.case_path / runscript).open() as input_file: to_write = input_file.read() runscript_path = cfg.icon_work / f"{runscript}.job" - with open(runscript_path, "w") as outf: + with runscript_path.open("w") as outf: outf.write( to_write.format(cfg=cfg, meteo=cfg.meteo, @@ -195,16 +194,15 @@ def main(cfg): async_error(cfg, part='COSMO') logging.info('COSMO analysis data for IC/BC') - dest_path = os.path.join(cfg.int2lm_input, 'meteo') + dest_path = cfg.int2lm_input / 'meteo' tools.create_dir(dest_path, "meteo input") source_nameformat = cfg.meteo['nameformat'] if cfg.meteo['prefix'] == 'lffd': # nested runs use cosmoart-output as meteo data # have to copy the *c.nc-file - src_file = os.path.join( - cfg.meteo['dir'], - cfg.startdate_sim.strftime(source_nameformat + 'c.nc')) + src_file = (cfg.meteo['dir'] / + cfg.startdate_sim.strftime(source_nameformat + 'c.nc')) tools.copy_file(src_file, dest_path, output_log=True) @@ -219,27 +217,22 @@ def main(cfg): num_steps = 0 meteo_dir = cfg.meteo['dir'] - subdir = os.path.join(meteo_dir, - cfg.startdate_sim.strftime('%y%m%d%H')) + subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, cfg.meteo['inc']): - dest_path = os.path.join(cfg.int2lm_input, 'meteo') - src_file = os.path.join(meteo_dir, - time.strftime(source_nameformat)) + dest_path = cfg.int2lm_input / 'meteo' + src_file = meteo_dir / time.strftime(source_nameformat) if cfg.meteo['prefix'] == 'efsf': if time == cfg.startdate_sim: - src_file = os.path.join(subdir, - 'eas' + time.strftime('%Y%m%d%H')) - if not os.path.isfile(src_file) and cfg.meteo.get('dir_alt') \ + src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) + if not src_file.exists() and cfg.meteo.get('dir_alt') \ is not None: meteo_dir = cfg.meteo['dir_alt'] - subdir = os.path.join( - meteo_dir, cfg.startdate_sim.strftime('%y%m%d%H')) - src_file = os.path.join( - subdir, 'eas' + time.strftime('%Y%m%d%H')) - dest_path = os.path.join(cfg.int2lm_input, 'meteo', - cfg.meteo['prefix'] + '00000000') + subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') + src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) + dest_path = cfg.int2lm_input / 'meteo' / ( + cfg.meteo['prefix'] + '00000000') else: td = time - cfg.startdate_sim - timedelta(hours=6 * num_steps) @@ -249,22 +242,19 @@ def main(cfg): days_total = str(td_total.days).zfill(2) hours_total = str(td_total.seconds // 3600).zfill(2) - src_file = os.path.join( - subdir, cfg.meteo['prefix'] + days + hours + '0000') - dest_path = os.path.join( - cfg.int2lm_input, 'meteo', cfg.meteo['prefix'] + - days_total + hours_total + '0000') + src_file = subdir / ( + cfg.meteo['prefix'] + days + hours + '0000') + dest_path = cfg.int2lm_input / 'meteo' / ( + cfg.meteo['prefix'] + days_total + hours_total + '0000') # Next time, change directory - checkdir = os.path.join(meteo_dir, - time.strftime('%y%m%d%H')) - if os.path.isdir(checkdir): + checkdir = meteo_dir / time.strftime('%y%m%d%H') + if checkdir.is_dir(): num_steps += 1 subdir = checkdir elif cfg.meteo.get('dir_alt') is not None: - checkdir = os.path.join(cfg.meteo['dir_alt'], - time.strftime('%y%m%d%H')) - if os.path.isdir(checkdir): + checkdir = cfg.meteo['dir_alt'] / time.strftime('%y%m%d%H') + if checkdir.is_dir(): num_steps += 1 subdir = checkdir meteo_dir = cfg.meteo['dir_alt'] @@ -272,12 +262,12 @@ def main(cfg): "Switching to other input directory from {} to {}" .format(cfg.meteo['dir'], cfg.meteo['dir_alt'])) - elif not os.path.exists(src_file): + elif not src_file.exists(): # special case for MeteoSwiss COSMO-7 data - archive = '/store/mch/msopr/owm/COSMO-7' + archive = Path('/store/mch/msopr/owm/COSMO-7') yy = time.strftime("%y") - path = '/'.join([archive, 'ANA' + yy]) - src_file = os.path.join(path, time.strftime(source_nameformat)) + path = archive / 'ANA' + yy + src_file = path / time.strftime(source_nameformat) # copy meteo file from project folder to tools.copy_file(src_file, dest_path, output_log=True) @@ -331,7 +321,7 @@ def main(cfg): ", ".join([i["fullname"] for i in inv_to_process]) + " data") - scratch_path = os.path.join(cfg.int2lm_input, 'icbc') + scratch_path = cfg.int2lm_input / 'icbc' tools.create_dir(scratch_path, "icbc input") for inv in inv_to_process: @@ -344,10 +334,9 @@ def main(cfg): cfg.enddate_sim, inc): logging.info(time) - filename = os.path.join( - inv["outdir"], p["suffix"] + "_" + - time.strftime("%Y%m%d%H") + ".nc") - if not os.path.exists(filename): + filename = inv["outdir"] / ( + p["suffix"] + "_" + time.strftime("%Y%m%d%H") + ".nc") + if not filename.exists(): logging.info(filename) try: to_call = getattr(tools, inv["executable"]) From 3221bb76d20c86c146d7c212ba4d02137cbc8693 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 5 Jan 2024 16:11:40 +0000 Subject: [PATCH 073/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 43474b0a..ce54b727 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -67,6 +67,7 @@ def async_error(cfg, part="This part"): # ... (previous code remains unchanged) + def main(cfg): """ **ICON and COSMO Data Preparation** @@ -145,7 +146,7 @@ def main(cfg): datafile_list_rest = [] datafile_list_chem = [] for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): meteo_file = cfg.icon_input_icbc / ( cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': @@ -228,7 +229,8 @@ def main(cfg): if not src_file.exists() and cfg.meteo.get('dir_alt') \ is not None: meteo_dir = cfg.meteo['dir_alt'] - subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') + subdir = meteo_dir / cfg.startdate_sim.strftime( + '%y%m%d%H') src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) dest_path = cfg.int2lm_input / 'meteo' / ( cfg.meteo['prefix'] + '00000000') @@ -241,10 +243,11 @@ def main(cfg): days_total = str(td_total.days).zfill(2) hours_total = str(td_total.seconds // 3600).zfill(2) - src_file = subdir / ( - cfg.meteo['prefix'] + days + hours + '0000') + src_file = subdir / (cfg.meteo['prefix'] + days + hours + + '0000') dest_path = cfg.int2lm_input / 'meteo' / ( - cfg.meteo['prefix'] + days_total + hours_total + '0000') + cfg.meteo['prefix'] + days_total + hours_total + + '0000') # Next time, change directory checkdir = meteo_dir / time.strftime('%y%m%d%H') @@ -252,7 +255,8 @@ def main(cfg): num_steps += 1 subdir = checkdir elif cfg.meteo.get('dir_alt') is not None: - checkdir = cfg.meteo['dir_alt'] / time.strftime('%y%m%d%H') + checkdir = cfg.meteo['dir_alt'] / time.strftime( + '%y%m%d%H') if checkdir.is_dir(): num_steps += 1 subdir = checkdir @@ -333,8 +337,9 @@ def main(cfg): cfg.enddate_sim, inc): logging.info(time) - filename = inv["outdir"] / ( - p["suffix"] + "_" + time.strftime("%Y%m%d%H") + ".nc") + filename = inv["outdir"] / (p["suffix"] + "_" + + time.strftime("%Y%m%d%H") + + ".nc") if not filename.exists(): logging.info(filename) try: From 7c4ab0702779981a73c8d1372f21c48a47c5a721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 5 Jan 2024 17:29:29 +0100 Subject: [PATCH 074/245] Fix for PosixPaths --- jobs/prepare_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 43474b0a..014f8194 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -151,9 +151,9 @@ def main(cfg): if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': chem_file = cfg.icon_input_icbc / ( cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) - datafile_list_chem.append(chem_file + cfg.chem['suffix']) - if meteo_file.endswith('00'): - datafile_list.append(meteo_file + cfg.meteo['suffix']) + datafile_list_chem.append(str(chem_file) + cfg.chem['suffix']) + if str(meteo_file).endswith('00'): + datafile_list.append(str(meteo_file) + cfg.meteo['suffix']) else: datafile_list_rest.append(meteo_file + cfg.meteo['suffix']) datafile_list = ' '.join([str(v) for v in datafile_list]) From 2a434963fd958b2e13add16575345847bfdf7353 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 8 Jan 2024 11:33:46 +0100 Subject: [PATCH 075/245] ADD: convenience function for slurm job info --- config.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config.py b/config.py index 5840da79..edf529a5 100644 --- a/config.py +++ b/config.py @@ -459,6 +459,20 @@ def submit(self, job_name, script, add_dep=None): # Can then be passed as add_dep keyword return result, job_id + def get_job_info(jobid, slurm_keys=['Elapsed']): + """Return information from slurm job as given by sacct + + All possible keys are given by `sacct --helpformat`""" + + # Get info from sacct + cmd = ['sacct', f'--format={','.join(slurm_keys)}', '--parsable', '-j', str(jobid)] + r = subprocess.run(cmd, capture_output=True) + + # Parse in a dictionnary before returning + # The inner most process should be the relevant one, hence the 1 index + slurm_info = r.stdout.split()[1].split(b'|') + return({k:v.decode() for k,v in zip(slurm_keys, slurm_info)}) + def check_submitted_job(self, script, result): exitcode = result.returncode if exitcode != 0: From 9956a7dc6616ddc7be7bf8654a5afbabcfe4634c Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 8 Jan 2024 11:38:10 +0100 Subject: [PATCH 076/245] REF: move get_job_info method to the end of the class definition --- config.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/config.py b/config.py index edf529a5..1a6d6417 100644 --- a/config.py +++ b/config.py @@ -459,20 +459,6 @@ def submit(self, job_name, script, add_dep=None): # Can then be passed as add_dep keyword return result, job_id - def get_job_info(jobid, slurm_keys=['Elapsed']): - """Return information from slurm job as given by sacct - - All possible keys are given by `sacct --helpformat`""" - - # Get info from sacct - cmd = ['sacct', f'--format={','.join(slurm_keys)}', '--parsable', '-j', str(jobid)] - r = subprocess.run(cmd, capture_output=True) - - # Parse in a dictionnary before returning - # The inner most process should be the relevant one, hence the 1 index - slurm_info = r.stdout.split()[1].split(b'|') - return({k:v.decode() for k,v in zip(slurm_keys, slurm_info)}) - def check_submitted_job(self, script, result): exitcode = result.returncode if exitcode != 0: @@ -511,6 +497,20 @@ def wait_for_previous(self): os.remove(job_file) os.remove(log_file) + def get_job_info(jobid, slurm_keys=['Elapsed']): + """Return information from slurm job as given by sacct + + All possible keys are given by `sacct --helpformat`""" + + # Get info from sacct + cmd = ['sacct', f'--format={','.join(slurm_keys)}', '--parsable', '-j', str(jobid)] + r = subprocess.run(cmd, capture_output=True) + + # Parse in a dictionnary before returning + # The inner most process should be the relevant one, hence the 1 index + slurm_info = r.stdout.split()[1].split(b'|') + return({k:v.decode() for k,v in zip(slurm_keys, slurm_info)}) + class InvalidWorkflowType(Exception): pass From 44b9cd0ec62caf69643d77150ca1e28136747334 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 8 Jan 2024 10:44:59 +0000 Subject: [PATCH 077/245] GitHub Action: Apply Pep8-formatting --- config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 8891c1a5..994028ea 100644 --- a/config.py +++ b/config.py @@ -541,13 +541,16 @@ def get_job_info(jobid, slurm_keys=['Elapsed']): All possible keys are given by `sacct --helpformat`""" # Get info from sacct - cmd = ['sacct', f'--format={','.join(slurm_keys)}', '--parsable', '-j', str(jobid)] + cmd = [ + 'sacct', f'--format={', '.join(slurm_keys)}', '--parsable', '-j', + str(jobid) + ] r = subprocess.run(cmd, capture_output=True) # Parse in a dictionnary before returning # The inner most process should be the relevant one, hence the 1 index slurm_info = r.stdout.split()[1].split(b'|') - return({k:v.decode() for k,v in zip(slurm_keys, slurm_info)}) + return ({k: v.decode() for k, v in zip(slurm_keys, slurm_info)}) class InvalidWorkflowType(Exception): From 192383d0c860b554069b8037ce588ad7718eeb3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 12:13:46 +0100 Subject: [PATCH 078/245] Add some docstrings --- config.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 994028ea..921ba54e 100644 --- a/config.py +++ b/config.py @@ -406,7 +406,6 @@ def finish_time_logging(self, job, launch_time): def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" - # Initial list of dependencies if add_dep is not None: if type(add_dep) is int: @@ -450,7 +449,6 @@ def get_dep_cmd(self, job_name, add_dep=None): def submit(self, job_name, script, add_dep=None, logfile=None): """Submit job with dependencies""" - script_path = Path(script) sbatch_cmd = ['sbatch', '--parsable'] if dep_cmd := self.get_dep_cmd(job_name, add_dep=add_dep): @@ -474,7 +472,9 @@ def submit(self, job_name, script, add_dep=None, logfile=None): return job_id def check_job(self, exitcode, logfile=None): - # In case of ICON-ART, ignore the "invalid pointer" error on successful run + """Check the exitcode returned by a job. + In case of ICON-ART, ignore the "invalid pointer" error on a successful run. + """ if logfile and tools.grep("ART: ", logfile)['success'] and \ tools.grep("free(): invalid pointer", logfile)['success'] and \ tools.grep("clean-up finished", logfile)['success']: @@ -484,6 +484,9 @@ def check_job(self, exitcode, logfile=None): raise RuntimeError(f"sbatch returned exitcode {exitcode}") def create_sbatch_script(self, job_name, log_file): + """Create an sbatch script to launch jobs individually. + Use run_chain.py arguments to submit those jobs. + """ script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}_{self.job_id}"', @@ -508,12 +511,11 @@ def create_sbatch_script(self, job_name, log_file): return job_file def wait_for_previous(self): - """wait for all jobs of the previous stage to be finished + """Wait for all jobs of the previous stage to be finished. Do this by submitting a fake job depending on all jobs from the 'previous' stage. """ - dep_ids = [] for ids in self.job_ids['previous'].values(): dep_ids.extend(ids) @@ -539,7 +541,6 @@ def get_job_info(jobid, slurm_keys=['Elapsed']): """Return information from slurm job as given by sacct All possible keys are given by `sacct --helpformat`""" - # Get info from sacct cmd = [ 'sacct', f'--format={', '.join(slurm_keys)}', '--parsable', '-j', From 8ded708e2a674c4ec69085a12af4b8e10c68d3ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 14:36:13 +0100 Subject: [PATCH 079/245] Fix quotes for f-strings --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 921ba54e..80e544ee 100644 --- a/config.py +++ b/config.py @@ -543,7 +543,7 @@ def get_job_info(jobid, slurm_keys=['Elapsed']): All possible keys are given by `sacct --helpformat`""" # Get info from sacct cmd = [ - 'sacct', f'--format={', '.join(slurm_keys)}', '--parsable', '-j', + "sacct", f"--format={', '.join(slurm_keys)}", "--parsable", "-j", str(jobid) ] r = subprocess.run(cmd, capture_output=True) From 1d8f3145e9cd2f1043ab1a03e7233feec139fea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 14:36:25 +0100 Subject: [PATCH 080/245] Add imports for art-global job --- jobs/prepare_art_global.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index cb182cd0..50038777 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -4,7 +4,10 @@ import os import logging import xarray as xr +import shutil +import subprocess from . import tools +from pathlib import Path from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging From f9bf445da6f1ce94fb1204db095476a941237c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 14:53:27 +0100 Subject: [PATCH 081/245] Add dependencies for all icon workflows --- workflows.yaml | 80 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/workflows.yaml b/workflows.yaml index 9c8b8480..de3c8666 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -1,14 +1,18 @@ cosmo: + features: + - restart + - spinup jobs: - prepare_data - int2lm - cosmo - post_cosmo + +cosmo-ghg: features: - restart + - tracers - spinup - -cosmo-ghg: jobs: - prepare_data - emissions @@ -19,12 +23,11 @@ cosmo-ghg: - post_int2lm - cosmo - post_cosmo - features: - - restart - - tracers - - spinup cosmo-art: + features: + - nesting + - spinup jobs: - prepare_data - emissions @@ -33,17 +36,14 @@ cosmo-art: - int2lm - cosmo - post_cosmo - features: - - nesting - - spinup icon: + features: + - restart jobs: - prepare_data - geosp - icon - features: - - restart dependencies: geosp: current: @@ -56,36 +56,78 @@ icon: - icon icon-seq: + features: + - restart jobs: - prepare_data - geosp - icon - features: - - restart icon-art: + features: + - restart jobs: - prepare_data - geosp - prepare_art - icon - features: - - restart + dependencies: + geosp: + current: + - prepare_data + prepare_art: + current: + - geosp + icon: + current: + - prepare_data + - geosp + - prepare_art + previous: + - icon icon-art-global: + features: + - restart jobs: - prepare_data - prepare_art_global - icon - features: - - restart + dependencies: + prepare_art_global: + current: + - prepare_data + icon: + current: + - prepare_data + - prepare_art_global + previous: + - icon icon-art-oem: + features: + - restart jobs: - prepare_data - geosp - prepare_art - prepare_art_oem - icon - features: - - restart + dependencies: + geosp: + current: + - prepare_data + prepare_art: + current: + - geosp + prepare_art_oem: + current: + - prepare_art + icon: + current: + - prepare_data + - geosp + - prepare_art + - prepare_art_oem + previous: + - icon From cc162dde7949c86f22243ab8a09840509ee5a9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 15:12:06 +0100 Subject: [PATCH 082/245] Fix pathname --- jobs/prepare_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index d36d6142..c8254909 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -156,7 +156,7 @@ def main(cfg): if str(meteo_file).endswith('00'): datafile_list.append(str(meteo_file) + cfg.meteo['suffix']) else: - datafile_list_rest.append(meteo_file + cfg.meteo['suffix']) + datafile_list_rest.append(str(meteo_file) + cfg.meteo['suffix']) datafile_list = ' '.join([str(v) for v in datafile_list]) datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) From 02e610a3479850b1186df7f7ad6f447a84022a0a Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 8 Jan 2024 14:12:35 +0000 Subject: [PATCH 083/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index c8254909..8b331939 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -156,7 +156,8 @@ def main(cfg): if str(meteo_file).endswith('00'): datafile_list.append(str(meteo_file) + cfg.meteo['suffix']) else: - datafile_list_rest.append(str(meteo_file) + cfg.meteo['suffix']) + datafile_list_rest.append( + str(meteo_file) + cfg.meteo['suffix']) datafile_list = ' '.join([str(v) for v in datafile_list]) datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) From ae3a814c429b72706000a08c646d09e6829502d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 17:14:31 +0100 Subject: [PATCH 084/245] Fix chunk calculation --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index dc6054b8..99ddc391 100755 --- a/run_chain.py +++ b/run_chain.py @@ -348,9 +348,9 @@ def restart_runs(cfg, force, resume): startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") job_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" - cfg.chunks.append(job_id) if enddate_sim > cfg.enddate: continue + cfg.chunks.append(job_id) for job_id in cfg.chunks: cfg.job_id = job_id From 75e306f2d1267c0eeb64f3df6eb26da5ef56d5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 17:56:48 +0100 Subject: [PATCH 085/245] Add icontools job --- jobs/__init__.py | 1 + jobs/icontools.py | 71 ++++++++++++++++++++++++++++++++++++++++++++ jobs/prepare_data.py | 54 --------------------------------- 3 files changed, 72 insertions(+), 54 deletions(-) create mode 100644 jobs/icontools.py diff --git a/jobs/__init__.py b/jobs/__init__.py index 094d28c7..50c5e305 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -7,6 +7,7 @@ from . import emissions from . import geosp from . import icon +from . import icontools from . import int2lm from . import obs_nudging from . import octe diff --git a/jobs/icontools.py b/jobs/icontools.py new file mode 100644 index 00000000..9107fdd4 --- /dev/null +++ b/jobs/icontools.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +from pathlib import Path +import logging +import shutil +import subprocess +from datetime import timedelta +import xarray as xr +from . import tools +from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond +from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging +from calendar import monthrange + + +def main(cfg): + launch_time = cfg.init_time_logging("icontools") + + #----------------------------------------------------- + # Create LBC datafile lists (each at 00 UTC and others) + #----------------------------------------------------- + datafile_list = [] + datafile_list_rest = [] + datafile_list_chem = [] + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + meteo_file = cfg.icon_input_icbc / ( + cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) + if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': + chem_file = cfg.icon_input_icbc / ( + cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) + datafile_list_chem.append(str(chem_file) + cfg.chem['suffix']) + if str(meteo_file).endswith('00'): + datafile_list.append(str(meteo_file) + cfg.meteo['suffix']) + else: + datafile_list_rest.append( + str(meteo_file) + cfg.meteo['suffix']) + datafile_list = ' '.join([str(v) for v in datafile_list]) + datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) + datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) + + #----------------------------------------------------- + # Write and submit runscripts + #----------------------------------------------------- + icontools_id = None + for runscript in cfg.icontools_runjobs: + with (cfg.case_path / runscript).open() as input_file: + to_write = input_file.read() + runscript_path = cfg.icon_work / f"{runscript}.job" + with runscript_path.open("w") as outf: + outf.write( + to_write.format(cfg=cfg, + meteo=cfg.meteo, + logfile=logfile, + logfile_finish=logfile_finish, + datafile_list=datafile_list, + datafile_list_rest=datafile_list_rest, + datafile_list_chem=datafile_list_chem)) + + # Submitting icontools runscripts sequentially + logging.info(f" Starting icontools runscript {runscript}.") + if icontools_id: + dependencies = (copy_id, icontools_id) + else: + dependencies = copy_id + icontools_id = cfg.submit(runscript, + runscript_path, + add_dep=dependencies) + + cfg.finish_time_logging("icontools", launch_time) diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index 8b331939..afdb1551 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -65,9 +65,6 @@ def async_error(cfg, part="This part"): f"{part} isn't ready for async execution yet") -# ... (previous code remains unchanged) - - def main(cfg): """ **ICON and COSMO Data Preparation** @@ -139,57 +136,6 @@ def main(cfg): copy_id = cfg.submit('prepare_data', script) - #----------------------------------------------------- - # Create LBC datafile lists (each at 00 UTC and others) - #----------------------------------------------------- - datafile_list = [] - datafile_list_rest = [] - datafile_list_chem = [] - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - meteo_file = cfg.icon_input_icbc / ( - cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) - if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': - chem_file = cfg.icon_input_icbc / ( - cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) - datafile_list_chem.append(str(chem_file) + cfg.chem['suffix']) - if str(meteo_file).endswith('00'): - datafile_list.append(str(meteo_file) + cfg.meteo['suffix']) - else: - datafile_list_rest.append( - str(meteo_file) + cfg.meteo['suffix']) - datafile_list = ' '.join([str(v) for v in datafile_list]) - datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) - datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) - - #----------------------------------------------------- - # Write and submit runscripts - #----------------------------------------------------- - icontools_id = None - for runscript in cfg.icontools_runjobs: - with (cfg.case_path / runscript).open() as input_file: - to_write = input_file.read() - runscript_path = cfg.icon_work / f"{runscript}.job" - with runscript_path.open("w") as outf: - outf.write( - to_write.format(cfg=cfg, - meteo=cfg.meteo, - logfile=logfile, - logfile_finish=logfile_finish, - datafile_list=datafile_list, - datafile_list_rest=datafile_list_rest, - datafile_list_chem=datafile_list_chem)) - - # Submitting icontools runscripts sequentially - logging.info(f" Starting icontools runscript {runscript}.") - if icontools_id: - dependencies = (copy_id, icontools_id) - else: - dependencies = copy_id - icontools_id = cfg.submit(runscript, - runscript_path, - add_dep=dependencies) - # If COSMO (and not ICON): else: async_error(cfg, part='COSMO') From c7d7c724c8b13e791302892cb02fd3883daec1cd Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 8 Jan 2024 16:57:17 +0000 Subject: [PATCH 086/245] GitHub Action: Apply Pep8-formatting --- jobs/icontools.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/jobs/icontools.py b/jobs/icontools.py index 9107fdd4..053d05b0 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -24,7 +24,7 @@ def main(cfg): datafile_list_rest = [] datafile_list_chem = [] for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): meteo_file = cfg.icon_input_icbc / ( cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': @@ -34,8 +34,7 @@ def main(cfg): if str(meteo_file).endswith('00'): datafile_list.append(str(meteo_file) + cfg.meteo['suffix']) else: - datafile_list_rest.append( - str(meteo_file) + cfg.meteo['suffix']) + datafile_list_rest.append(str(meteo_file) + cfg.meteo['suffix']) datafile_list = ' '.join([str(v) for v in datafile_list]) datafile_list_rest = ' '.join([str(v) for v in datafile_list_rest]) datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) @@ -65,7 +64,7 @@ def main(cfg): else: dependencies = copy_id icontools_id = cfg.submit(runscript, - runscript_path, - add_dep=dependencies) + runscript_path, + add_dep=dependencies) cfg.finish_time_logging("icontools", launch_time) From c3024a04b9a35c68ab81cdcf47e17481a166753b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 17:59:50 +0100 Subject: [PATCH 087/245] Add icontools to dependencies --- workflows.yaml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/workflows.yaml b/workflows.yaml index de3c8666..e699d426 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -42,15 +42,20 @@ icon: - restart jobs: - prepare_data + - icontools - geosp - icon dependencies: - geosp: + icontools: current: - prepare_data + geosp: + current: + - icontools icon: current: - prepare_data + - icontools - geosp previous: - icon @@ -60,6 +65,7 @@ icon-seq: - restart jobs: - prepare_data + - icontools - geosp - icon @@ -68,19 +74,24 @@ icon-art: - restart jobs: - prepare_data + - icontools - geosp - prepare_art - icon dependencies: - geosp: + icontools: current: - prepare_data + geosp: + current: + - icontools prepare_art: current: - geosp icon: current: - prepare_data + - icontools - geosp - prepare_art previous: @@ -109,14 +120,18 @@ icon-art-oem: - restart jobs: - prepare_data + - icontools - geosp - prepare_art - prepare_art_oem - icon dependencies: - geosp: + icontools: current: - prepare_data + geosp: + current: + - icontools prepare_art: current: - geosp @@ -126,6 +141,7 @@ icon-art-oem: icon: current: - prepare_data + - icontools - geosp - prepare_art - prepare_art_oem From 767ec08ea74ee07d41582660a2d1d703527c3711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 8 Jan 2024 18:01:31 +0100 Subject: [PATCH 088/245] Modify docstrings --- jobs/geosp.py | 3 ++- jobs/icontools.py | 7 +++++++ jobs/prepare_data.py | 6 ------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jobs/geosp.py b/jobs/geosp.py index 5d1929ab..b94ac296 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -11,7 +11,8 @@ def main(cfg): """ - Add GEOSP to all meteo files + The constant variable 'GEOSP' is added to the files not containing it + using python-cdo bindings. """ prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("geosp") diff --git a/jobs/icontools.py b/jobs/icontools.py index 053d05b0..c88df1af 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -15,6 +15,13 @@ def main(cfg): + """ + - Submit the runscript for the DWD ICON tools to remap the meteorological files. + - All runscripts specified in ``cfg.icontools_runjobs`` are submitted. + - The meteorological files are read from the original input directory + (``cfg.input_root_meteo``), and the remapped meteorological files are saved + in the input folder on scratch (``cfg.icon_input/icbc``). + """ launch_time = cfg.init_time_logging("icontools") #----------------------------------------------------- diff --git a/jobs/prepare_data.py b/jobs/prepare_data.py index afdb1551..53a4f942 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_data.py @@ -73,13 +73,7 @@ def main(cfg): copying meteorological files, and handling specific data processing for each model. **ICON:** - - Create directories ``cfg.icon_input_icbc`` and ``cfg.icon_work``. - - Submit the runscript for the DWD ICON tools to remap the meteorological files. - - All runscripts specified in ``cfg.icontools_runjobs`` are submitted. - - The meteorological files are read from the original input directory (``cfg.input_root_meteo``), - and the remapped meteorological files are saved in the input folder on scratch (``cfg.icon_input/icbc``). - - The constant variable 'GEOSP' is added to the files not containing it using python-cdo bindings. **COSMO:** From fd8b603e9ccf92563f55fabca87d2e67eac2a338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 11:07:14 +0100 Subject: [PATCH 089/245] Some code cleanup --- config.py | 15 +++++++-------- jobs/prepare_art_global.py | 6 +++--- run_chain.py | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/config.py b/config.py index 80e544ee..f7a82446 100644 --- a/config.py +++ b/config.py @@ -1,7 +1,6 @@ import subprocess import os import yaml -import logging from jobs import tools from pathlib import Path @@ -408,13 +407,13 @@ def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" # Initial list of dependencies if add_dep is not None: - if type(add_dep) is int: + if isinstance(add_dep, int): dep_id_list = [add_dep] else: try: dep_id_list = list(add_dep) except TypeError: - print(f'add_dep must be an iterable') + print("add_dep must be an iterable") else: dep_id_list = [] @@ -461,7 +460,7 @@ def submit(self, job_name, script, add_dep=None, logfile=None): job_id = int(result.stdout) print(f' └── Submitted batch job {job_id}') - if not job_name in self.job_ids['current']: + if job_name not in self.job_ids['current']: self.job_ids['current'][job_name] = [job_id] else: self.job_ids['current'][job_name].append(job_id) @@ -490,9 +489,9 @@ def create_sbatch_script(self, job_name, log_file): script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}_{self.job_id}"', - f'#SBATCH --nodes=1', + '#SBATCH --nodes=1', f'#SBATCH --output={log_file}', - f'#SBATCH --open-mode=append', + '#SBATCH --open-mode=append', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', @@ -524,8 +523,8 @@ def wait_for_previous(self): log_file = self.case_root / 'wait.log' dep_str = ':'.join(map(str, dep_ids)) script_lines = [ - '#!/usr/bin/env bash', f'#SBATCH --job-name="wait"', - f'#SBATCH --nodes=1', f'#SBATCH --output={log_file}', + '#!/usr/bin/env bash', '#SBATCH --job-name="wait"', + '#SBATCH --nodes=1', f'#SBATCH --output={log_file}', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 50038777..e6e68924 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -8,7 +8,7 @@ import subprocess from . import tools from pathlib import Path -from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond +from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond # noqa: F401 from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging @@ -153,8 +153,8 @@ def main(cfg): else: # -- Check the extension of tracer variables in the restart file - ds_restart = xr.open_dataset(cfg.restart_file) - tracer_name = cfg.species2restart[0] + ds_restart = xr.open_dataset(cfg.restart_file) # noqa: F841 + tracer_name = cfg.species2restart[0] # noqa: F841 # FIXME: # var_restart = [ # IndexError: list index out of range diff --git a/run_chain.py b/run_chain.py index 99ddc391..c0692a97 100755 --- a/run_chain.py +++ b/run_chain.py @@ -239,7 +239,7 @@ def run_chunk(cfg, force, resume): # Submit the job script = cfg.create_sbatch_script(job, logfile) - job_id = cfg.submit(job, script) + cfg.submit(job, script) # wait for previous chunk to be done cfg.wait_for_previous() @@ -295,7 +295,7 @@ def run_chunk(cfg, force, resume): exitcode = 0 try_count = 0 - except: + except Exception: subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( job, cfg.job_id) logging.exception(subject) From d7033a57a377edadc8105702dc2aaef58f44cc36 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Tue, 9 Jan 2024 11:16:33 +0100 Subject: [PATCH 090/245] fix(icontools): remove unsused packages + fix dependency logic - The `icontools` job already depends on `prepapre_data`, no need to add a `copy_id` in the dependencies (which was anyways undefinied in this scope). - The name of the first argument of `cfg.submit()` must be 'icontools' so that all the sub job ids are associated with the icontools job and are taken into account for other jobs depending on it (icon). Otherwise, icon will find no dependency in `cfg.job_ids['current']['icontools']`. - `cfg.get_job_info()` should be used later in `run_chain.py` to monitor time for all async jobs. Currently the reported time is only the submission time. --- jobs/icontools.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/jobs/icontools.py b/jobs/icontools.py index c88df1af..9c1d63f5 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -1,17 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os -from pathlib import Path import logging -import shutil -import subprocess -from datetime import timedelta -import xarray as xr from . import tools -from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond -from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -from calendar import monthrange def main(cfg): @@ -49,7 +40,7 @@ def main(cfg): #----------------------------------------------------- # Write and submit runscripts #----------------------------------------------------- - icontools_id = None + dep_id = None for runscript in cfg.icontools_runjobs: with (cfg.case_path / runscript).open() as input_file: to_write = input_file.read() @@ -66,12 +57,6 @@ def main(cfg): # Submitting icontools runscripts sequentially logging.info(f" Starting icontools runscript {runscript}.") - if icontools_id: - dependencies = (copy_id, icontools_id) - else: - dependencies = copy_id - icontools_id = cfg.submit(runscript, - runscript_path, - add_dep=dependencies) + dep_id = cfg.submit('icontools', runscript_path, add_dep=dep_id) cfg.finish_time_logging("icontools", launch_time) From d2dd23d1d46c062a67f3c1324ae781e9c44835b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 11:17:12 +0100 Subject: [PATCH 091/245] Make prepare_art_global dependent on previous icon --- jobs/prepare_art_global.py | 70 ++------------------------------------ workflows.yaml | 2 ++ 2 files changed, 5 insertions(+), 67 deletions(-) diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index e6e68924..366d47a7 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -12,82 +12,18 @@ from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -def set_cfg_variables(cfg): - if cfg.workflow_name.startswith('cosmo'): - cfg.int2lm_root = cfg.chain_root / 'int2lm' - cfg.int2lm_input = cfg.int2lm_root / 'input' - elif cfg.workflow_name.startswith('icon'): - cfg.icon_base = cfg.chain_root / 'icon' - cfg.icon_input = cfg.icon_base / 'input' - cfg.icon_input_icbc = cfg.icon_input / 'icbc' - cfg.icon_work = cfg.icon_base / 'run' - cfg.icon_output = cfg.icon_base / 'output' - cfg.icon_output_reduced = cfg.icon_base / 'output_reduced' - cfg.icon_restart_out = cfg.icon_base / 'restart' - cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' - cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' - - cfg.input_files_scratch = {} - for dsc, file in cfg.input_files.items(): - cfg.input_files[dsc] = (p := Path(file)) - cfg.input_files_scratch[dsc] = cfg.icon_input / p.name - - cfg.create_vars_from_dicts() - - cfg.ini_datetime_string = cfg.startdate.strftime('%Y-%m-%dT%H:00:00Z') - cfg.end_datetime_string = cfg.enddate.strftime('%Y-%m-%dT%H:00:00Z') - - if cfg.workflow_name == 'icon-art-oem': - cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime( - '%Y%m%d_%H') - - if cfg.workflow_name == 'icon-art-global': - # Nudge type (global or nothing) - cfg.nudge_type = 2 if cfg.era5_global_nudging else 0 - # Time step for global nudging in seconds - cfg.nudging_step_seconds = cfg.nudging_step * 3600 - # Prescribed initial conditions for CH4, CO and/or OH - cfg.iart_init_gas = 4 if cfg.species_inicond else 0 - - if cfg.lrestart == '.TRUE.': - cfg.restart_filename = 'restart_atm_DOM01.nc' - cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename - cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename - - cfg.job_ids['current']['prepare_data'] = [] - - def main(cfg): """ - This function prepares the data for global ICON-ART simulations - by downloading necessary meteorological and chemical data, - creating the inicond file, and handling global nudging - conditions. The workflow includes fetching ERA5 data, processing it, - and creating initial conditions for tracers (such as CH4, CO, and OH). - - The main steps performed by this function include: - 1. Downloading ERA5 data and creating the inicond file if specified in the - configuration. - 2. Creating initial conditions for tracers (CH4, CO, OH) if needed, - including handling restart scenarios. - 3. Handling global nudging conditions by downloading and processing ERA5 and - CAMS data if configured. - - Note - ---- - - The function utilizes external scripts for processing ERA5 data - (`icon_era5_inicond.sh`, `icon_era5_nudging.sh`) and CAMS data - (`icon_cams_nudging.sh`). - - The `tools` module provides various utility functions used in the workflow. + Prepare global ICON-ART simulations. Parameters ---------- cfg : Config Object holding all user-configuration parameters as attributes. """ - set_cfg_variables(cfg) + prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art_global") - logging.info("Preprae ICON-ART for global simulations") + logging.info("Prepare ICON-ART for global simulations") # -- Download ERA5 data and create the inicond file if cfg.era5_inicond and cfg.lrestart == '.FALSE.': diff --git a/workflows.yaml b/workflows.yaml index e699d426..1f584286 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -108,6 +108,8 @@ icon-art-global: prepare_art_global: current: - prepare_data + previous: + - icon icon: current: - prepare_data From 3ce57d652900168f26fa69b2b19f54cd72d5efde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 11:27:47 +0100 Subject: [PATCH 092/245] Fix import --- jobs/prepare_art_global.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 366d47a7..0b0cce50 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -6,7 +6,7 @@ import xarray as xr import shutil import subprocess -from . import tools +from . import tools, prepare_data from pathlib import Path from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond # noqa: F401 from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging From 99b96cf7beda6d143c9fa9198ee5be75a6366ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 11:49:24 +0100 Subject: [PATCH 093/245] Explicitly set async variable --- cases/icon-art-global-test/config.yaml | 1 + cases/icon-art-oem-test/config.yaml | 1 + cases/icon-seq-test/config.yaml | 3 ++- cases/icon-test/config.yaml | 1 + config.py | 4 +--- workflows.yaml | 9 --------- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cases/icon-art-global-test/config.yaml b/cases/icon-art-global-test/config.yaml index 5b49bd38..e0499c0b 100644 --- a/cases/icon-art-global-test/config.yaml +++ b/cases/icon-art-global-test/config.yaml @@ -1,6 +1,7 @@ # Configuration file for the 'icon-art-global-test' case with ICON workflow: icon-art-global +is_async: True constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index 06d678d4..c71f8c01 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -1,6 +1,7 @@ # Configuration file for the 'icon-art-oem-test' case with ICON workflow: icon-art-oem +is_async: True constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml index cee58297..2a65ded4 100644 --- a/cases/icon-seq-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -1,6 +1,7 @@ # Configuration file for the 'icon-test' case with ICON -workflow: icon-seq +workflow: icon +is_async: False constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index 261a0871..2592725a 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -1,6 +1,7 @@ # Configuration file for the 'icon-async-test' case with ICON workflow: icon +is_async: True constraint: gpu run_on: cpu compute_queue: normal diff --git a/config.py b/config.py index f7a82446..70471138 100644 --- a/config.py +++ b/config.py @@ -190,7 +190,7 @@ def set_node_info(self): "It should be either 'gpu' or 'mc'.") def set_workflow(self): - """set workflow and async attr, initiate job ids dict""" + """set workflow and initiate job ids dict""" # If a workflow name is specified, load from workflows.yaml if isinstance(self.workflow, str): with open('workflows.yaml') as file: @@ -204,8 +204,6 @@ def set_workflow(self): "Invalid workflow type. Must be either a string or a dictionary." ) - self.is_async = 'dependencies' in self.workflow - # Initiate empty job ids dictionnary so that it can be filled in later self.job_ids = {'current': {}, 'previous': {}} diff --git a/workflows.yaml b/workflows.yaml index 1f584286..13c824ae 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -60,15 +60,6 @@ icon: previous: - icon -icon-seq: - features: - - restart - jobs: - - prepare_data - - icontools - - geosp - - icon - icon-art: features: - restart From 07dac892bf9376c398aa9c0f35ed1d21f930f7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 11:59:22 +0100 Subject: [PATCH 094/245] Add logfile for icontools --- jobs/icontools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/icontools.py b/jobs/icontools.py index 9c1d63f5..3e54aa1e 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -14,6 +14,8 @@ def main(cfg): in the input folder on scratch (``cfg.icon_input/icbc``). """ launch_time = cfg.init_time_logging("icontools") + logfile = cfg.log_working_dir / "icontools" + logfile_finish = cfg.log_finished_dir / "icontools" #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) From 7169901b8d577ba4006db4ddad670ea0ebf39a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 12:02:08 +0100 Subject: [PATCH 095/245] Set variables for icontools --- jobs/icontools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/icontools.py b/jobs/icontools.py index 3e54aa1e..2cfc3482 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import logging -from . import tools +from . import tools, prepare_data def main(cfg): @@ -13,6 +13,7 @@ def main(cfg): (``cfg.input_root_meteo``), and the remapped meteorological files are saved in the input folder on scratch (``cfg.icon_input/icbc``). """ + prepare_data.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("icontools") logfile = cfg.log_working_dir / "icontools" logfile_finish = cfg.log_finished_dir / "icontools" From 0c9f906768c595d09b29ea242f560f9021b473cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 12:17:22 +0100 Subject: [PATCH 096/245] Revert "Explicitly set async variable" This reverts commit 99b96cf7beda6d143c9fa9198ee5be75a6366ee8. --- cases/icon-art-global-test/config.yaml | 1 - cases/icon-art-oem-test/config.yaml | 1 - cases/icon-seq-test/config.yaml | 3 +-- cases/icon-test/config.yaml | 1 - config.py | 4 +++- workflows.yaml | 9 +++++++++ 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cases/icon-art-global-test/config.yaml b/cases/icon-art-global-test/config.yaml index e0499c0b..5b49bd38 100644 --- a/cases/icon-art-global-test/config.yaml +++ b/cases/icon-art-global-test/config.yaml @@ -1,7 +1,6 @@ # Configuration file for the 'icon-art-global-test' case with ICON workflow: icon-art-global -is_async: True constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index c71f8c01..06d678d4 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -1,7 +1,6 @@ # Configuration file for the 'icon-art-oem-test' case with ICON workflow: icon-art-oem -is_async: True constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml index 2a65ded4..cee58297 100644 --- a/cases/icon-seq-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -1,7 +1,6 @@ # Configuration file for the 'icon-test' case with ICON -workflow: icon -is_async: False +workflow: icon-seq constraint: gpu run_on: cpu compute_queue: normal diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index 2592725a..261a0871 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -1,7 +1,6 @@ # Configuration file for the 'icon-async-test' case with ICON workflow: icon -is_async: True constraint: gpu run_on: cpu compute_queue: normal diff --git a/config.py b/config.py index 70471138..f7a82446 100644 --- a/config.py +++ b/config.py @@ -190,7 +190,7 @@ def set_node_info(self): "It should be either 'gpu' or 'mc'.") def set_workflow(self): - """set workflow and initiate job ids dict""" + """set workflow and async attr, initiate job ids dict""" # If a workflow name is specified, load from workflows.yaml if isinstance(self.workflow, str): with open('workflows.yaml') as file: @@ -204,6 +204,8 @@ def set_workflow(self): "Invalid workflow type. Must be either a string or a dictionary." ) + self.is_async = 'dependencies' in self.workflow + # Initiate empty job ids dictionnary so that it can be filled in later self.job_ids = {'current': {}, 'previous': {}} diff --git a/workflows.yaml b/workflows.yaml index 13c824ae..1f584286 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -60,6 +60,15 @@ icon: previous: - icon +icon-seq: + features: + - restart + jobs: + - prepare_data + - icontools + - geosp + - icon + icon-art: features: - restart From f3e9e47bcda7f141ebf7a9e7a2fa257eacf51b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 12:25:10 +0100 Subject: [PATCH 097/245] Define sequential workflow in icon-seq-test --- cases/icon-seq-test/config.yaml | 11 +++++++++-- workflows.yaml | 9 --------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml index cee58297..0c341b17 100644 --- a/cases/icon-seq-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -1,6 +1,13 @@ -# Configuration file for the 'icon-test' case with ICON +# Configuration file for the 'icon-seq-test' case with ICON -workflow: icon-seq +workflow: + features: + - restart + jobs: + - prepare_data + - icontools + - geosp + - icon constraint: gpu run_on: cpu compute_queue: normal diff --git a/workflows.yaml b/workflows.yaml index 1f584286..13c824ae 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -60,15 +60,6 @@ icon: previous: - icon -icon-seq: - features: - - restart - jobs: - - prepare_data - - icontools - - geosp - - icon - icon-art: features: - restart From 8040ee7a1c9100b9e74cc100326ee09089aac5e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 12:29:57 +0100 Subject: [PATCH 098/245] Make cosmo-ghg workflow async --- workflows.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/workflows.yaml b/workflows.yaml index 13c824ae..b2e74e63 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -23,6 +23,34 @@ cosmo-ghg: - post_int2lm - cosmo - post_cosmo + dependencies: + emissions: + current: + - prepare_data + biofluxes: + current: + - prepare_data + oem: + current: + - prepare_data + online_vprm: + current: + - prepare_data + int2lm: + current: + - prepare_data + - emissions + - biofluxes + - oem + - online_vprm + post_int2lm: + current: + - int2lm + cosmo: + current: + - post_int2lm + previous: + - cosmo cosmo-art: features: From 2acb4bf3e9f6316770d65e64f170c16b88886bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 12:33:32 +0100 Subject: [PATCH 099/245] Add info about (a)sync mode --- run_chain.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/run_chain.py b/run_chain.py index c0692a97..f47fac0b 100755 --- a/run_chain.py +++ b/run_chain.py @@ -491,6 +491,11 @@ def main(): f"Starting chain for case {casename} and workflow {cfg.workflow_name}" ) + if cfg.is_async: + print("Running the Processing Chain in asynchronous mode.") + else: + print("Running the Processing Chain in sequential mode.") + if cfg.logging: launch_time = cfg.init_time_logging('chain') From 3d6e9914e2fd7b1043bfa3462d8acda6bbb75d62 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Tue, 9 Jan 2024 13:02:08 +0100 Subject: [PATCH 100/245] fix: also launch waiting job when some workflow jobs failed --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index f7a82446..d8f94ff1 100644 --- a/config.py +++ b/config.py @@ -528,7 +528,7 @@ def wait_for_previous(self): f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', - f'#SBATCH --dependency=afterok:{dep_str}', '', '# Do nothing', + f'#SBATCH --dependency=afterany:{dep_str}', '', '# Do nothing', 'exit 0' ] with open(job_file, mode='w') as wait_job: From 52f06a16d702b67ac0453ec66da275bae5deba4d Mon Sep 17 00:00:00 2001 From: Matthieu Date: Tue, 9 Jan 2024 13:03:39 +0100 Subject: [PATCH 101/245] add: other keys to default job info dict --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index d8f94ff1..8bc9fc8f 100644 --- a/config.py +++ b/config.py @@ -536,8 +536,8 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) - def get_job_info(jobid, slurm_keys=['Elapsed']): - """Return information from slurm job as given by sacct + def get_job_info(jobid, slurm_keys=['JobName', 'Elapsed', 'ExitCode']): + """Return info dict from slurm job as given by sacct All possible keys are given by `sacct --helpformat`""" # Get info from sacct From 276a7565d61966516e00f72364e54e3951467709 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Tue, 9 Jan 2024 13:13:20 +0100 Subject: [PATCH 102/245] fix: empty current job ids before each chunk --- run_chain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run_chain.py b/run_chain.py index f47fac0b..f03d7e8c 100755 --- a/run_chain.py +++ b/run_chain.py @@ -223,6 +223,9 @@ def run_chunk(cfg, force, resume): cfg.convert_gas = True if cfg.is_async: + # Empty curent job ids + cfg.job_ids['current'] = {} + # Submit current chunk for job in cfg.jobs: if (cfg.log_finished_dir / job).exists() and not force: @@ -243,6 +246,7 @@ def run_chunk(cfg, force, resume): # wait for previous chunk to be done cfg.wait_for_previous() + # cycle cfg.job_ids['previous'] = cfg.job_ids['current'] else: From 1528d186898821e761f2af17a2c5d50cf7072361 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 9 Jan 2024 12:14:11 +0000 Subject: [PATCH 103/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index f03d7e8c..b177ce1e 100755 --- a/run_chain.py +++ b/run_chain.py @@ -225,7 +225,7 @@ def run_chunk(cfg, force, resume): if cfg.is_async: # Empty curent job ids cfg.job_ids['current'] = {} - + # Submit current chunk for job in cfg.jobs: if (cfg.log_finished_dir / job).exists() and not force: @@ -246,7 +246,7 @@ def run_chunk(cfg, force, resume): # wait for previous chunk to be done cfg.wait_for_previous() - + # cycle cfg.job_ids['previous'] = cfg.job_ids['current'] else: From 9b87be8934a5eb252e2754170e8b596500ef681e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 13:22:37 +0100 Subject: [PATCH 104/245] Split prepare_data into cosmo and icon --- jobs/__init__.py | 3 +- jobs/biofluxes.py | 4 +- jobs/emissions.py | 4 +- jobs/geosp.py | 4 +- jobs/icon.py | 6 +- jobs/icontools.py | 4 +- jobs/int2lm.py | 4 +- jobs/prepare_art.py | 4 +- jobs/prepare_art_global.py | 14 +- jobs/prepare_art_oem.py | 35 +--- jobs/prepare_cosmo.py | 208 ++++++++++++++++++++++ jobs/{prepare_data.py => prepare_icon.py} | 86 ++++----- jobs/tools/__init__.py | 2 +- 13 files changed, 278 insertions(+), 100 deletions(-) create mode 100644 jobs/prepare_cosmo.py rename jobs/{prepare_data.py => prepare_icon.py} (80%) diff --git a/jobs/__init__.py b/jobs/__init__.py index 50c5e305..1934831f 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -19,6 +19,7 @@ from . import prepare_art from . import prepare_art_oem from . import prepare_art_global -from . import prepare_data +from . import prepare_cosmo +from . import prepare_icon from . import reduce_output from . import verify_chain diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index b33d1524..bed680e9 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -3,7 +3,7 @@ import os import logging -from . import tools, prepare_data +from . import tools, prepare_icon def main(cfg): @@ -18,7 +18,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.check_model(cfg, 'cosmo-ghg') - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) scratch_path = os.path.join(cfg.int2lm_input, 'vprm') diff --git a/jobs/emissions.py b/jobs/emissions.py index a4cdf207..0647ee21 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -4,7 +4,7 @@ import os import logging -from . import tools, prepare_data +from . import tools, prepare_icon def main(cfg): @@ -30,7 +30,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) dest_prefix = "emis_" if not isinstance(cfg.emissions['dir'], list): diff --git a/jobs/geosp.py b/jobs/geosp.py index b94ac296..b0843bc8 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -6,7 +6,7 @@ import logging import xarray as xr import numpy as np -from . import tools, prepare_data +from . import tools, prepare_icon def main(cfg): @@ -14,7 +14,7 @@ def main(cfg): The constant variable 'GEOSP' is added to the files not containing it using python-cdo bindings. """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("geosp") logging.info("Add GEOSP to all meteo files") diff --git a/jobs/icon.py b/jobs/icon.py index a2932b8b..360df7b8 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import logging -from . import tools, prepare_data +from . import tools, prepare_icon, prepare_art_global, prepare_art_oem def main(cfg): @@ -31,7 +31,9 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) + prepare_art_global.set_cfg_variables(cfg) + prepare_art_oem.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("icon") logfile = cfg.log_working_dir / "icon" diff --git a/jobs/icontools.py b/jobs/icontools.py index 2cfc3482..e93d0ddd 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import logging -from . import tools, prepare_data +from . import tools, prepare_icon def main(cfg): @@ -13,7 +13,7 @@ def main(cfg): (``cfg.input_root_meteo``), and the remapped meteorological files are saved in the input folder on scratch (``cfg.icon_input/icbc``). """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("icontools") logfile = cfg.log_working_dir / "icontools" logfile_finish = cfg.log_finished_dir / "icontools" diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 519370d0..336bb422 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -7,7 +7,7 @@ import subprocess import pytz from datetime import datetime, timedelta -from . import tools, prepare_data +from . import tools, prepare_cosmo def set_cfg_variables(cfg): @@ -46,7 +46,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_data.set_cfg_variables(cfg) + prepare_cosmo.set_cfg_variables(cfg) set_cfg_variables(cfg) # Total number of processes diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 94fae342..eac45644 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -5,14 +5,14 @@ import logging import xarray as xr import numpy as np -from . import tools, prepare_data +from . import tools, prepare_icon def main(cfg): """ Add Q (copy of QV) and/or PS to initial file """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") logging.info('Add Q (copy of QV) and/or PS to initial file') diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 0b0cce50..3c31f677 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -6,12 +6,21 @@ import xarray as xr import shutil import subprocess -from . import tools, prepare_data +from . import tools, prepare_icon from pathlib import Path from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond # noqa: F401 from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging +def set_cfg_variables(cfg): + # Nudge type (global or nothing) + cfg.nudge_type = 2 if cfg.era5_global_nudging else 0 + # Time step for global nudging in seconds + cfg.nudging_step_seconds = cfg.nudging_step * 3600 + # Prescribed initial conditions for CH4, CO and/or OH + cfg.iart_init_gas = 4 if cfg.species_inicond else 0 + + def main(cfg): """ Prepare global ICON-ART simulations. @@ -21,7 +30,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) + set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art_global") logging.info("Prepare ICON-ART for global simulations") diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 3ed16d19..f5ebcfa3 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -4,38 +4,19 @@ import os import logging import xarray as xr -from . import tools, prepare_data +from . import tools, prepare_icon +def set_cfg_variables(cfg): + cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime( + '%Y%m%d_%H') + def main(cfg): """ - This function sets up the necessary configuration variables, logs the - initialization time, and merges chemical tracers with meteorological files if - OEM (Online Emission Model) is enabled. The merging process involves - combining initial and lateral boundary condition (IC and LBC) files for the - ICON-ART model. - - **IC Merging (Initial Conditions):** - For the initial time step, it merges the IC files by: - - Combining meteorological and chemical files. - - Adjusting variables such as converting LNPS to PS for surface pressure. - - Saving the merged dataset to a new NetCDF file. - - Renaming and cleaning up the files. - - **LBC Merging (Lateral Boundary Conditions):** - For subsequent time steps, it merges the LBC files by: - - Combining meteorological and chemical files. - - Adjusting variables such as converting LNPS to PS for surface pressure - and renaming specific variables. - - Saving the merged dataset to a new NetCDF file. - - Renaming and cleaning up the files. - - This function utilizes the :func:`prepare_data.set_cfg_variables` - function to set configuration variables, :mod:`xarray` (xr) for handling - NetCDF files, and various utility functions from the :mod:`tools` module for - file operations. + ICON-ART-OEM preparations """ - prepare_data.set_cfg_variables(cfg) + prepare_icon.set_cfg_variables(cfg) + set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") logging.info('Merging IC and LBC') diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py new file mode 100644 index 00000000..3f751d08 --- /dev/null +++ b/jobs/prepare_cosmo.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pathlib import Path +import logging +from datetime import timedelta +from . import tools + + +def set_cfg_variables(cfg): + cfg.int2lm_root = cfg.chain_root / 'int2lm' + cfg.int2lm_input = cfg.int2lm_root / 'input' + + +def main(cfg): + """ + **COSMO Data Preparation** + + This function prepares input data for COSMO simulations by creating necessary directories, + copying meteorological files, and handling specific data processing. + + - Copy meteorological files to **int2lm** input. + - Create the necessary directory ``cfg.int2lm_input/meteo``. + - Copy meteorological files from the project directory (``cfg.meteo['dir']/cfg.meteo['prefix']YYYYMMDDHH``) + to the int2lm input folder on scratch (``cfg.int2lm_input/meteo``). + - For nested runs (meteorological files are COSMO output: ``cfg.meteo['prefix'] == 'lffd'``), + also copy the ``*c.nc``-file with constant parameters. + + Parameters + ---------- + cfg : Config + Object holding all user-configuration parameters as attributes. + + Raises + ------ + RuntimeError + If any subprocess returns a non-zero exit code during execution. + """ + set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("prepare_cosmo") + + logging.info('COSMO analysis data for IC/BC') + + dest_path = cfg.int2lm_input / 'meteo' + tools.create_dir(dest_path, "meteo input") + + source_nameformat = cfg.meteo['nameformat'] + if cfg.meteo['prefix'] == 'lffd': + # nested runs use cosmoart-output as meteo data + # have to copy the *c.nc-file + src_file = (cfg.meteo['dir'] / + cfg.startdate_sim.strftime(source_nameformat + 'c.nc')) + + tools.copy_file(src_file, dest_path, output_log=True) + + logging.info("Copied constant-param file from {} to {}".format( + src_file, dest_path)) + + # extend nameformat with ending to match cosmo-output + source_nameformat += '.nc' + + if cfg.meteo['prefix'] == 'efsf': + source_nameformat = cfg.meteo['prefix'] + '%y%m%d%H' + + num_steps = 0 + meteo_dir = cfg.meteo['dir'] + subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + dest_path = cfg.int2lm_input / 'meteo' + src_file = meteo_dir / time.strftime(source_nameformat) + + if cfg.meteo['prefix'] == 'efsf': + if time == cfg.startdate_sim: + src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) + if not src_file.exists() and cfg.meteo.get('dir_alt') \ + is not None: + meteo_dir = cfg.meteo['dir_alt'] + subdir = meteo_dir / cfg.startdate_sim.strftime( + '%y%m%d%H') + src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) + dest_path = cfg.int2lm_input / 'meteo' / ( + cfg.meteo['prefix'] + '00000000') + else: + td = time - cfg.startdate_sim - timedelta(hours=6 * + num_steps) + days = str(td.days).zfill(2) + hours = str(td.seconds // 3600).zfill(2) + td_total = time - cfg.startdate_sim + days_total = str(td_total.days).zfill(2) + hours_total = str(td_total.seconds // 3600).zfill(2) + + src_file = subdir / (cfg.meteo['prefix'] + days + hours + + '0000') + dest_path = cfg.int2lm_input / 'meteo' / ( + cfg.meteo['prefix'] + days_total + hours_total + + '0000') + + # Next time, change directory + checkdir = meteo_dir / time.strftime('%y%m%d%H') + if checkdir.is_dir(): + num_steps += 1 + subdir = checkdir + elif cfg.meteo.get('dir_alt') is not None: + checkdir = cfg.meteo['dir_alt'] / time.strftime( + '%y%m%d%H') + if checkdir.is_dir(): + num_steps += 1 + subdir = checkdir + meteo_dir = cfg.meteo['dir_alt'] + logging.info( + "Switching to other input directory from {} to {}" + .format(cfg.meteo['dir'], + cfg.meteo['dir_alt'])) + elif not src_file.exists(): + # special case for MeteoSwiss COSMO-7 data + archive = Path('/store/mch/msopr/owm/COSMO-7') + yy = time.strftime("%y") + path = archive / 'ANA' + yy + src_file = path / time.strftime(source_nameformat) + + # copy meteo file from project folder to + tools.copy_file(src_file, dest_path, output_log=True) + + logging.info("Copied file from {} to {}".format( + src_file, dest_path)) + + # Other IC/BC data + inv_to_process = [] + if cfg.workflow_name == 'cosmo-ghg': + try: + CAMS = dict(fullname="CAMS", + nickname="cams", + executable="cams4int2cosmo", + indir=cfg.cams['dir_orig'], + outdir=cfg.cams['dir_proc'], + param=[{ + 'inc': cfg.cams['inc'], + 'suffix': cfg.cams['suffix'] + }]) + inv_to_process.append(CAMS) + except AttributeError: + pass + try: + CT = dict(fullname="CarbonTracker", + nickname="ct", + executable="ctnoaa4int2cosmo", + indir=cfg.ct_dir_orig, + outdir=cfg.ct_dir_proc, + param=cfg.ct_parameters) + inv_to_process.append(CT) + except AttributeError: + pass + elif cfg.workflow_name == 'cosmo-art': + try: + MOZART = dict(fullname='MOZART', + nickname='mozart', + executable='mozart2int2lm', + indir=cfg.mozart_file_orig, + outdir=cfg.mozart_dir_proc, + param=[{ + 'inc': cfg.mozart_inc, + 'suffix': cfg.mozart_prefix + }]) + inv_to_process.append(MOZART) + except AttributeError: + pass + + if cfg.workflow_name == 'cosmo-ghg' or cfg.workflow_name == 'cosmo-art': + logging.info("Processing " + + ", ".join([i["fullname"] + for i in inv_to_process]) + " data") + + scratch_path = cfg.int2lm_input / 'icbc' + tools.create_dir(scratch_path, "icbc input") + + for inv in inv_to_process: + logging.info(inv["fullname"] + " files") + tools.create_dir(inv["outdir"], "processed " + inv["fullname"]) + + for p in inv["param"]: + inc = p["inc"] + for time in tools.iter_hours(cfg.startdate_sim, + cfg.enddate_sim, inc): + logging.info(time) + + filename = inv["outdir"] / (p["suffix"] + "_" + + time.strftime("%Y%m%d%H") + + ".nc") + if not filename.exists(): + logging.info(filename) + try: + to_call = getattr(tools, inv["executable"]) + to_call.main(time, inv["indir"], inv["outdir"], + p) + except: + logging.error("Preprocessing " + + inv["fullname"] + " data failed") + raise + + # copy to (temporary) run input directory + tools.copy_file(filename, + scratch_path, + output_log=True) + + logging.info("OK") + + cfg.finish_time_logging("prepare_cosmo", launch_time) diff --git a/jobs/prepare_data.py b/jobs/prepare_icon.py similarity index 80% rename from jobs/prepare_data.py rename to jobs/prepare_icon.py index 53a4f942..3c5e0bbe 100644 --- a/jobs/prepare_data.py +++ b/jobs/prepare_icon.py @@ -1,62 +1,39 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os from pathlib import Path import logging -import shutil -import subprocess from datetime import timedelta -import xarray as xr from . import tools -from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond -from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -from calendar import monthrange def set_cfg_variables(cfg): - if cfg.workflow_name.startswith('cosmo'): - cfg.int2lm_root = cfg.chain_root / 'int2lm' - cfg.int2lm_input = cfg.int2lm_root / 'input' - elif cfg.workflow_name.startswith('icon'): - cfg.icon_base = cfg.chain_root / 'icon' - cfg.icon_input = cfg.icon_base / 'input' - cfg.icon_input_icbc = cfg.icon_input / 'icbc' - cfg.icon_work = cfg.icon_base / 'run' - cfg.icon_output = cfg.icon_base / 'output' - cfg.icon_output_reduced = cfg.icon_base / 'output_reduced' - cfg.icon_restart_out = cfg.icon_base / 'restart' - cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' - cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' - - cfg.input_files_scratch = {} - for dsc, file in cfg.input_files.items(): - cfg.input_files[dsc] = (p := Path(file)) - cfg.input_files_scratch[dsc] = cfg.icon_input / p.name - - cfg.create_vars_from_dicts() - - cfg.ini_datetime_string = cfg.startdate.strftime('%Y-%m-%dT%H:00:00Z') - cfg.end_datetime_string = cfg.enddate.strftime('%Y-%m-%dT%H:00:00Z') - - if cfg.workflow_name == 'icon-art-oem': - cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime( - '%Y%m%d_%H') - - if cfg.workflow_name == 'icon-art-global': - # Nudge type (global or nothing) - cfg.nudge_type = 2 if cfg.era5_global_nudging else 0 - # Time step for global nudging in seconds - cfg.nudging_step_seconds = cfg.nudging_step * 3600 - # Prescribed initial conditions for CH4, CO and/or OH - cfg.iart_init_gas = 4 if cfg.species_inicond else 0 - - if cfg.lrestart == '.TRUE.': - cfg.restart_filename = 'restart_atm_DOM01.nc' - cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename - cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename - - cfg.job_ids['current']['prepare_data'] = [] + cfg.icon_base = cfg.chain_root / 'icon' + cfg.icon_input = cfg.icon_base / 'input' + cfg.icon_input_icbc = cfg.icon_input / 'icbc' + cfg.icon_work = cfg.icon_base / 'run' + cfg.icon_output = cfg.icon_base / 'output' + cfg.icon_output_reduced = cfg.icon_base / 'output_reduced' + cfg.icon_restart_out = cfg.icon_base / 'restart' + cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' + cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' + + cfg.input_files_scratch = {} + for dsc, file in cfg.input_files.items(): + cfg.input_files[dsc] = (p := Path(file)) + cfg.input_files_scratch[dsc] = cfg.icon_input / p.name + + cfg.create_vars_from_dicts() + + cfg.ini_datetime_string = cfg.startdate.strftime('%Y-%m-%dT%H:00:00Z') + cfg.end_datetime_string = cfg.enddate.strftime('%Y-%m-%dT%H:00:00Z') + + if cfg.lrestart == '.TRUE.': + cfg.restart_filename = 'restart_atm_DOM01.nc' + cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename + cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename + + cfg.job_ids['current']['prepare_icon'] = [] def async_error(cfg, part="This part"): @@ -95,7 +72,7 @@ def main(cfg): If any subprocess returns a non-zero exit code during execution. """ set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("prepare_data") + launch_time = cfg.init_time_logging("prepare_icon") if cfg.workflow_name.startswith('icon'): logging.info('ICON input data (IC/BC)') @@ -107,15 +84,14 @@ def main(cfg): tools.create_dir(cfg.icon_restart_out, "icon_restart_out") # Set logfile - logfile = cfg.log_working_dir / 'prepare_data' - logfile_finish = cfg.log_finished_dir / 'prepare_data' + logfile = cfg.log_working_dir / 'prepare_icon' # Copy input files to scratch script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', f'#SBATCH --account={cfg.compute_account}', - f'#SBATCH --time=00:10:00', + '#SBATCH --time=00:10:00', f'#SBATCH --partition={cfg.compute_queue}', f'#SBATCH --constraint={cfg.constraint}', '#SBATCH --nodes=1', f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', @@ -128,7 +104,7 @@ def main(cfg): with (script := cfg.icon_work / 'copy_input.job').open('w') as f: f.write('\n'.join(script_lines)) - copy_id = cfg.submit('prepare_data', script) + cfg.submit('prepare_icon', script) # If COSMO (and not ICON): else: @@ -299,4 +275,4 @@ def main(cfg): logging.info("OK") - cfg.finish_time_logging("prepare_data", launch_time) + cfg.finish_time_logging("prepare_icon", launch_time) diff --git a/jobs/tools/__init__.py b/jobs/tools/__init__.py index 01b8f3e5..b21a6569 100644 --- a/jobs/tools/__init__.py +++ b/jobs/tools/__init__.py @@ -420,7 +420,7 @@ def check_job_completion(log_finished_dir, job, waittime=3000): log_finished_dir : directory for logfiles of finished jobs - job: string of job name, e.g. "prepare_data" + job: string of job name, e.g. "prepare_icon" waittime : time to wait (factor of .1 second) Defaults to 3000 (300 seconds) From d4443bb8b88c7704e84ee30528a4190e0b0cfd70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 13:25:40 +0100 Subject: [PATCH 105/245] Remove unused imports --- jobs/int2lm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 336bb422..77f14c8d 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -5,8 +5,7 @@ import logging import shutil import subprocess -import pytz -from datetime import datetime, timedelta +from datetime import datetime from . import tools, prepare_cosmo From 39f0993cac07f2543fb5fe1e42681bc9d07bc4eb Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 9 Jan 2024 12:26:11 +0000 Subject: [PATCH 106/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_art_oem.py | 6 ++-- jobs/prepare_cosmo.py | 78 ++++++++++++++++++----------------------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index f5ebcfa3..a1122c1d 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -8,8 +8,8 @@ def set_cfg_variables(cfg): - cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime( - '%Y%m%d_%H') + cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime('%Y%m%d_%H') + def main(cfg): """ @@ -88,4 +88,4 @@ def main(cfg): logging.info( "Added chemical tracer to file {}".format(merged_file)) - cfg.finish_time_logging("prepare_art_oem", launch_time) \ No newline at end of file + cfg.finish_time_logging("prepare_art_oem", launch_time) diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 3f751d08..0e26a3c0 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -8,8 +8,8 @@ def set_cfg_variables(cfg): - cfg.int2lm_root = cfg.chain_root / 'int2lm' - cfg.int2lm_input = cfg.int2lm_root / 'input' + cfg.int2lm_root = cfg.chain_root / 'int2lm' + cfg.int2lm_input = cfg.int2lm_root / 'input' def main(cfg): @@ -66,7 +66,7 @@ def main(cfg): meteo_dir = cfg.meteo['dir'] subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): + cfg.meteo['inc']): dest_path = cfg.int2lm_input / 'meteo' src_file = meteo_dir / time.strftime(source_nameformat) @@ -76,14 +76,12 @@ def main(cfg): if not src_file.exists() and cfg.meteo.get('dir_alt') \ is not None: meteo_dir = cfg.meteo['dir_alt'] - subdir = meteo_dir / cfg.startdate_sim.strftime( - '%y%m%d%H') + subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) - dest_path = cfg.int2lm_input / 'meteo' / ( - cfg.meteo['prefix'] + '00000000') + dest_path = cfg.int2lm_input / 'meteo' / (cfg.meteo['prefix'] + + '00000000') else: - td = time - cfg.startdate_sim - timedelta(hours=6 * - num_steps) + td = time - cfg.startdate_sim - timedelta(hours=6 * num_steps) days = str(td.days).zfill(2) hours = str(td.seconds // 3600).zfill(2) td_total = time - cfg.startdate_sim @@ -91,10 +89,9 @@ def main(cfg): hours_total = str(td_total.seconds // 3600).zfill(2) src_file = subdir / (cfg.meteo['prefix'] + days + hours + - '0000') + '0000') dest_path = cfg.int2lm_input / 'meteo' / ( - cfg.meteo['prefix'] + days_total + hours_total + - '0000') + cfg.meteo['prefix'] + days_total + hours_total + '0000') # Next time, change directory checkdir = meteo_dir / time.strftime('%y%m%d%H') @@ -102,16 +99,14 @@ def main(cfg): num_steps += 1 subdir = checkdir elif cfg.meteo.get('dir_alt') is not None: - checkdir = cfg.meteo['dir_alt'] / time.strftime( - '%y%m%d%H') + checkdir = cfg.meteo['dir_alt'] / time.strftime('%y%m%d%H') if checkdir.is_dir(): num_steps += 1 subdir = checkdir meteo_dir = cfg.meteo['dir_alt'] logging.info( - "Switching to other input directory from {} to {}" - .format(cfg.meteo['dir'], - cfg.meteo['dir_alt'])) + "Switching to other input directory from {} to {}". + format(cfg.meteo['dir'], cfg.meteo['dir_alt'])) elif not src_file.exists(): # special case for MeteoSwiss COSMO-7 data archive = Path('/store/mch/msopr/owm/COSMO-7') @@ -122,8 +117,7 @@ def main(cfg): # copy meteo file from project folder to tools.copy_file(src_file, dest_path, output_log=True) - logging.info("Copied file from {} to {}".format( - src_file, dest_path)) + logging.info("Copied file from {} to {}".format(src_file, dest_path)) # Other IC/BC data inv_to_process = [] @@ -143,32 +137,32 @@ def main(cfg): pass try: CT = dict(fullname="CarbonTracker", - nickname="ct", - executable="ctnoaa4int2cosmo", - indir=cfg.ct_dir_orig, - outdir=cfg.ct_dir_proc, - param=cfg.ct_parameters) + nickname="ct", + executable="ctnoaa4int2cosmo", + indir=cfg.ct_dir_orig, + outdir=cfg.ct_dir_proc, + param=cfg.ct_parameters) inv_to_process.append(CT) except AttributeError: pass elif cfg.workflow_name == 'cosmo-art': try: MOZART = dict(fullname='MOZART', - nickname='mozart', - executable='mozart2int2lm', - indir=cfg.mozart_file_orig, - outdir=cfg.mozart_dir_proc, - param=[{ - 'inc': cfg.mozart_inc, - 'suffix': cfg.mozart_prefix - }]) + nickname='mozart', + executable='mozart2int2lm', + indir=cfg.mozart_file_orig, + outdir=cfg.mozart_dir_proc, + param=[{ + 'inc': cfg.mozart_inc, + 'suffix': cfg.mozart_prefix + }]) inv_to_process.append(MOZART) except AttributeError: pass if cfg.workflow_name == 'cosmo-ghg' or cfg.workflow_name == 'cosmo-art': logging.info("Processing " + - ", ".join([i["fullname"] + ", ".join([i["fullname"] for i in inv_to_process]) + " data") scratch_path = cfg.int2lm_input / 'icbc' @@ -181,27 +175,23 @@ def main(cfg): for p in inv["param"]: inc = p["inc"] for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, inc): + cfg.enddate_sim, inc): logging.info(time) - filename = inv["outdir"] / (p["suffix"] + "_" + - time.strftime("%Y%m%d%H") + - ".nc") + filename = inv["outdir"] / ( + p["suffix"] + "_" + time.strftime("%Y%m%d%H") + ".nc") if not filename.exists(): logging.info(filename) try: to_call = getattr(tools, inv["executable"]) - to_call.main(time, inv["indir"], inv["outdir"], - p) + to_call.main(time, inv["indir"], inv["outdir"], p) except: - logging.error("Preprocessing " + - inv["fullname"] + " data failed") + logging.error("Preprocessing " + inv["fullname"] + + " data failed") raise # copy to (temporary) run input directory - tools.copy_file(filename, - scratch_path, - output_log=True) + tools.copy_file(filename, scratch_path, output_log=True) logging.info("OK") From d65606e0b397f4b450e62bad3913fcf2ccd5c4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 13:26:59 +0100 Subject: [PATCH 107/245] Rename prepare_data job in workflows --- workflows.yaml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/workflows.yaml b/workflows.yaml index b2e74e63..a7d8d56d 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -3,7 +3,7 @@ cosmo: - restart - spinup jobs: - - prepare_data + - prepare_cosmo - int2lm - cosmo - post_cosmo @@ -14,7 +14,7 @@ cosmo-ghg: - tracers - spinup jobs: - - prepare_data + - prepare_cosmo - emissions - biofluxes - oem @@ -26,19 +26,19 @@ cosmo-ghg: dependencies: emissions: current: - - prepare_data + - prepare_cosmo biofluxes: current: - - prepare_data + - prepare_cosmo oem: current: - - prepare_data + - prepare_cosmo online_vprm: current: - - prepare_data + - prepare_cosmo int2lm: current: - - prepare_data + - prepare_cosmo - emissions - biofluxes - oem @@ -57,7 +57,7 @@ cosmo-art: - nesting - spinup jobs: - - prepare_data + - prepare_cosmo - emissions - obs_nudging - photo_rate @@ -69,20 +69,20 @@ icon: features: - restart jobs: - - prepare_data + - prepare_icon - icontools - geosp - icon dependencies: icontools: current: - - prepare_data + - prepare_icon geosp: current: - icontools icon: current: - - prepare_data + - prepare_icon - icontools - geosp previous: @@ -92,7 +92,7 @@ icon-art: features: - restart jobs: - - prepare_data + - prepare_icon - icontools - geosp - prepare_art @@ -100,7 +100,7 @@ icon-art: dependencies: icontools: current: - - prepare_data + - prepare_icon geosp: current: - icontools @@ -109,7 +109,7 @@ icon-art: - geosp icon: current: - - prepare_data + - prepare_icon - icontools - geosp - prepare_art @@ -120,18 +120,18 @@ icon-art-global: features: - restart jobs: - - prepare_data + - prepare_icon - prepare_art_global - icon dependencies: prepare_art_global: current: - - prepare_data + - prepare_icon previous: - icon icon: current: - - prepare_data + - prepare_icon - prepare_art_global previous: - icon @@ -140,7 +140,7 @@ icon-art-oem: features: - restart jobs: - - prepare_data + - prepare_icon - icontools - geosp - prepare_art @@ -149,7 +149,7 @@ icon-art-oem: dependencies: icontools: current: - - prepare_data + - prepare_icon geosp: current: - icontools @@ -161,7 +161,7 @@ icon-art-oem: - prepare_art icon: current: - - prepare_data + - prepare_icon - icontools - geosp - prepare_art From cfeb7e90c3aa64197bdb267edb76ae98b469c513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 13:27:38 +0100 Subject: [PATCH 108/245] Rename prepare_data in icon-seq case --- cases/icon-seq-test/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml index 0c341b17..341bb646 100644 --- a/cases/icon-seq-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -4,7 +4,7 @@ workflow: features: - restart jobs: - - prepare_data + - prepare_icon - icontools - geosp - icon From 7a7a7ad353acda519cf111058f1477032bd03ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 14:29:33 +0100 Subject: [PATCH 109/245] Set additional cfg variables in prepare_icon --- jobs/icon.py | 2 +- jobs/prepare_art_global.py | 10 -- jobs/prepare_art_oem.py | 5 - jobs/prepare_icon.py | 204 +++---------------------------------- 4 files changed, 14 insertions(+), 207 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 360df7b8..3cb8d059 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -68,6 +68,6 @@ def main(cfg): script.write_text(script_str) # Submit run script - job_id = cfg.submit('icon', script, logfile=logfile) + cfg.submit('icon', script, logfile=logfile) cfg.finish_time_logging("icon", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 3c31f677..dad6a306 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -12,15 +12,6 @@ from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging -def set_cfg_variables(cfg): - # Nudge type (global or nothing) - cfg.nudge_type = 2 if cfg.era5_global_nudging else 0 - # Time step for global nudging in seconds - cfg.nudging_step_seconds = cfg.nudging_step * 3600 - # Prescribed initial conditions for CH4, CO and/or OH - cfg.iart_init_gas = 4 if cfg.species_inicond else 0 - - def main(cfg): """ Prepare global ICON-ART simulations. @@ -31,7 +22,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_icon.set_cfg_variables(cfg) - set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art_global") logging.info("Prepare ICON-ART for global simulations") diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index a1122c1d..4d6be5f9 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -7,16 +7,11 @@ from . import tools, prepare_icon -def set_cfg_variables(cfg): - cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime('%Y%m%d_%H') - - def main(cfg): """ ICON-ART-OEM preparations """ prepare_icon.set_cfg_variables(cfg) - set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") logging.info('Merging IC and LBC') diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 3c5e0bbe..410d926c 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -3,7 +3,6 @@ from pathlib import Path import logging -from datetime import timedelta from . import tools @@ -32,34 +31,26 @@ def set_cfg_variables(cfg): cfg.restart_filename = 'restart_atm_DOM01.nc' cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename + + # Nudge type (global or nothing) + cfg.nudge_type = 2 if hasattr(cfg, 'era5') and cfg.era5_global_nudging else 0 + # Time step for global nudging in seconds + cfg.nudging_step_seconds = cfg.nudging_step * 3600 if hasattr(cfg, 'nudging_step') else None + # Prescribed initial conditions for CH4, CO and/or OH + cfg.iart_init_gas = 4 if hasattr(cfg, 'species_inicond') and cfg.species_inicond else 0 - cfg.job_ids['current']['prepare_icon'] = [] - - -def async_error(cfg, part="This part"): - if cfg.is_async: - raise NotImplementedError( - f"{part} isn't ready for async execution yet") + cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime('%Y%m%d_%H') + cfg.job_ids['current']['prepare_icon'] = [] def main(cfg): """ - **ICON and COSMO Data Preparation** + **ICON Data Preparation** - This function prepares input data for ICON and COSMO simulations by creating necessary directories, - copying meteorological files, and handling specific data processing for each model. + This function prepares input data for ICON simulations by creating necessary directories, + copying meteorological files, and handling specific data processing. - **ICON:** - - Create directories ``cfg.icon_input_icbc`` and ``cfg.icon_work``. - - **COSMO:** - - - Copy meteorological files to **int2lm** input. - - Create the necessary directory ``cfg.int2lm_input/meteo``. - - Copy meteorological files from the project directory (``cfg.meteo['dir']/cfg.meteo['prefix']YYYYMMDDHH``) - to the int2lm input folder on scratch (``cfg.int2lm_input/meteo``). - - For nested runs (meteorological files are COSMO output: ``cfg.meteo['prefix'] == 'lffd'``), - also copy the ``*c.nc``-file with constant parameters. + - Create working directories and copy input files Parameters ---------- @@ -106,173 +97,4 @@ def main(cfg): cfg.submit('prepare_icon', script) - # If COSMO (and not ICON): - else: - async_error(cfg, part='COSMO') - logging.info('COSMO analysis data for IC/BC') - - dest_path = cfg.int2lm_input / 'meteo' - tools.create_dir(dest_path, "meteo input") - - source_nameformat = cfg.meteo['nameformat'] - if cfg.meteo['prefix'] == 'lffd': - # nested runs use cosmoart-output as meteo data - # have to copy the *c.nc-file - src_file = (cfg.meteo['dir'] / - cfg.startdate_sim.strftime(source_nameformat + 'c.nc')) - - tools.copy_file(src_file, dest_path, output_log=True) - - logging.info("Copied constant-param file from {} to {}".format( - src_file, dest_path)) - - # extend nameformat with ending to match cosmo-output - source_nameformat += '.nc' - - if cfg.meteo['prefix'] == 'efsf': - source_nameformat = cfg.meteo['prefix'] + '%y%m%d%H' - - num_steps = 0 - meteo_dir = cfg.meteo['dir'] - subdir = meteo_dir / cfg.startdate_sim.strftime('%y%m%d%H') - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - dest_path = cfg.int2lm_input / 'meteo' - src_file = meteo_dir / time.strftime(source_nameformat) - - if cfg.meteo['prefix'] == 'efsf': - if time == cfg.startdate_sim: - src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) - if not src_file.exists() and cfg.meteo.get('dir_alt') \ - is not None: - meteo_dir = cfg.meteo['dir_alt'] - subdir = meteo_dir / cfg.startdate_sim.strftime( - '%y%m%d%H') - src_file = subdir / ('eas' + time.strftime('%Y%m%d%H')) - dest_path = cfg.int2lm_input / 'meteo' / ( - cfg.meteo['prefix'] + '00000000') - else: - td = time - cfg.startdate_sim - timedelta(hours=6 * - num_steps) - days = str(td.days).zfill(2) - hours = str(td.seconds // 3600).zfill(2) - td_total = time - cfg.startdate_sim - days_total = str(td_total.days).zfill(2) - hours_total = str(td_total.seconds // 3600).zfill(2) - - src_file = subdir / (cfg.meteo['prefix'] + days + hours + - '0000') - dest_path = cfg.int2lm_input / 'meteo' / ( - cfg.meteo['prefix'] + days_total + hours_total + - '0000') - - # Next time, change directory - checkdir = meteo_dir / time.strftime('%y%m%d%H') - if checkdir.is_dir(): - num_steps += 1 - subdir = checkdir - elif cfg.meteo.get('dir_alt') is not None: - checkdir = cfg.meteo['dir_alt'] / time.strftime( - '%y%m%d%H') - if checkdir.is_dir(): - num_steps += 1 - subdir = checkdir - meteo_dir = cfg.meteo['dir_alt'] - logging.info( - "Switching to other input directory from {} to {}" - .format(cfg.meteo['dir'], - cfg.meteo['dir_alt'])) - elif not src_file.exists(): - # special case for MeteoSwiss COSMO-7 data - archive = Path('/store/mch/msopr/owm/COSMO-7') - yy = time.strftime("%y") - path = archive / 'ANA' + yy - src_file = path / time.strftime(source_nameformat) - - # copy meteo file from project folder to - tools.copy_file(src_file, dest_path, output_log=True) - - logging.info("Copied file from {} to {}".format( - src_file, dest_path)) - - # Other IC/BC data - inv_to_process = [] - if cfg.workflow_name == 'cosmo-ghg': - try: - CAMS = dict(fullname="CAMS", - nickname="cams", - executable="cams4int2cosmo", - indir=cfg.cams['dir_orig'], - outdir=cfg.cams['dir_proc'], - param=[{ - 'inc': cfg.cams['inc'], - 'suffix': cfg.cams['suffix'] - }]) - inv_to_process.append(CAMS) - except AttributeError: - pass - try: - CT = dict(fullname="CarbonTracker", - nickname="ct", - executable="ctnoaa4int2cosmo", - indir=cfg.ct_dir_orig, - outdir=cfg.ct_dir_proc, - param=cfg.ct_parameters) - inv_to_process.append(CT) - except AttributeError: - pass - elif cfg.workflow_name == 'cosmo-art': - try: - MOZART = dict(fullname='MOZART', - nickname='mozart', - executable='mozart2int2lm', - indir=cfg.mozart_file_orig, - outdir=cfg.mozart_dir_proc, - param=[{ - 'inc': cfg.mozart_inc, - 'suffix': cfg.mozart_prefix - }]) - inv_to_process.append(MOZART) - except AttributeError: - pass - - if cfg.workflow_name == 'cosmo-ghg' or cfg.workflow_name == 'cosmo-art': - logging.info("Processing " + - ", ".join([i["fullname"] - for i in inv_to_process]) + " data") - - scratch_path = cfg.int2lm_input / 'icbc' - tools.create_dir(scratch_path, "icbc input") - - for inv in inv_to_process: - logging.info(inv["fullname"] + " files") - tools.create_dir(inv["outdir"], "processed " + inv["fullname"]) - - for p in inv["param"]: - inc = p["inc"] - for time in tools.iter_hours(cfg.startdate_sim, - cfg.enddate_sim, inc): - logging.info(time) - - filename = inv["outdir"] / (p["suffix"] + "_" + - time.strftime("%Y%m%d%H") + - ".nc") - if not filename.exists(): - logging.info(filename) - try: - to_call = getattr(tools, inv["executable"]) - to_call.main(time, inv["indir"], inv["outdir"], - p) - except: - logging.error("Preprocessing " + - inv["fullname"] + " data failed") - raise - - # copy to (temporary) run input directory - tools.copy_file(filename, - scratch_path, - output_log=True) - - logging.info("OK") - cfg.finish_time_logging("prepare_icon", launch_time) From 7df686dc5721f6af8c02c2196cc3c52de2bdf0e3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 9 Jan 2024 13:30:30 +0000 Subject: [PATCH 110/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_icon.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 410d926c..32b6d9ee 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -31,18 +31,22 @@ def set_cfg_variables(cfg): cfg.restart_filename = 'restart_atm_DOM01.nc' cfg.restart_file = cfg.icon_restart_in / cfg.restart_filename cfg.restart_file_scratch = cfg.icon_work / cfg.restart_filename - - # Nudge type (global or nothing) - cfg.nudge_type = 2 if hasattr(cfg, 'era5') and cfg.era5_global_nudging else 0 - # Time step for global nudging in seconds - cfg.nudging_step_seconds = cfg.nudging_step * 3600 if hasattr(cfg, 'nudging_step') else None - # Prescribed initial conditions for CH4, CO and/or OH - cfg.iart_init_gas = 4 if hasattr(cfg, 'species_inicond') and cfg.species_inicond else 0 + + # Nudge type (global or nothing) + cfg.nudge_type = 2 if hasattr(cfg, + 'era5') and cfg.era5_global_nudging else 0 + # Time step for global nudging in seconds + cfg.nudging_step_seconds = cfg.nudging_step * 3600 if hasattr( + cfg, 'nudging_step') else None + # Prescribed initial conditions for CH4, CO and/or OH + cfg.iart_init_gas = 4 if hasattr( + cfg, 'species_inicond') and cfg.species_inicond else 0 cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime('%Y%m%d_%H') cfg.job_ids['current']['prepare_icon'] = [] + def main(cfg): """ **ICON Data Preparation** From f55331d077199853383785a9e5ea26c692928d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 14:43:38 +0100 Subject: [PATCH 111/245] Time logging for all jobs --- jobs/biofluxes.py | 2 ++ jobs/check_output.py | 3 +++ jobs/cosmo.py | 3 +++ jobs/emissions.py | 3 +++ jobs/geosp.py | 2 +- jobs/int2lm.py | 4 +++- jobs/obs_nudging.py | 3 +++ jobs/octe.py | 2 ++ jobs/oem.py | 3 +++ jobs/online_vprm.py | 2 ++ jobs/photo_rate.py | 3 +++ jobs/post_cosmo.py | 3 +++ jobs/post_int2lm.py | 3 +++ jobs/prepare_art_global.py | 2 +- jobs/reduce_output.py | 3 +++ jobs/verify_chain.py | 3 +++ 16 files changed, 41 insertions(+), 3 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index bed680e9..7fc0628e 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -19,6 +19,7 @@ def main(cfg): """ tools.check_model(cfg, 'cosmo-ghg') prepare_icon.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("biofluxes") scratch_path = os.path.join(cfg.int2lm_input, 'vprm') @@ -44,3 +45,4 @@ def main(cfg): logging.error( "Splitting or copying of GPP or/and RA files to scratch failed." ) + cfg.finish_time_logging("biofluxes", launch_time) \ No newline at end of file diff --git a/jobs/check_output.py b/jobs/check_output.py index ccf45595..51e70178 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -713,6 +713,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + launch_time = cfg.init_time_logging("check_output") date = dt.datetime.today() to_print = """check_output @@ -863,6 +864,8 @@ def main(cfg): if 'ERROR' in f.read(): raise RuntimeError('Logfile containing errors! See %s' % logfile) + cfg.finish_time_logging("check_output", launch_time) + if __name__ == '__main__': args = sys.argv diff --git a/jobs/cosmo.py b/jobs/cosmo.py index c9552208..399d8f57 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -77,6 +77,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("cosmo") logfile = os.path.join(cfg.log_working_dir, "cosmo") logfile_finish = os.path.join(cfg.log_finished_dir, "cosmo") @@ -240,3 +241,5 @@ def main(cfg): exitcode = result.returncode if exitcode != 0: raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + + cfg.finish_time_logging("cosmo", launch_time) \ No newline at end of file diff --git a/jobs/emissions.py b/jobs/emissions.py index 0647ee21..b3dd32bc 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -31,6 +31,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_icon.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("emissions") dest_prefix = "emis_" if not isinstance(cfg.emissions['dir'], list): @@ -67,3 +68,5 @@ def main(cfg): # (NF90_CHAR) (needed for int2lm to work) if cfg.workflow_name.startswith('cosmo'): tools.string2char.main(dest_path) + + cfg.finish_time_logging("emissions", launch_time) \ No newline at end of file diff --git a/jobs/geosp.py b/jobs/geosp.py index b0843bc8..b15d2332 100644 --- a/jobs/geosp.py +++ b/jobs/geosp.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import os -from pathlib import Path +from pathlib import Path # noqa: F401 import logging import xarray as xr import numpy as np diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 77f14c8d..0293d123 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -13,7 +13,6 @@ def set_cfg_variables(cfg): cfg.int2lm_run = cfg.chain_root / 'int2lm' / 'run' cfg.int2lm_output = cfg.chain_root / 'int2lm' / 'output' - def main(cfg): """Setup the namelist for int2lm and submit the job to the queue. @@ -47,6 +46,7 @@ def main(cfg): """ prepare_cosmo.set_cfg_variables(cfg) set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("int2lm") # Total number of processes np_tot = cfg.int2lm['np_x'] * cfg.int2lm['np_y'] @@ -164,3 +164,5 @@ def main(cfg): exitcode = result.returncode if exitcode != 0: raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + + cfg.finish_time_logging("int2lm", launch_time) \ No newline at end of file diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index b5539728..a199e42a 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -27,6 +27,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + launch_time = cfg.init_time_logging("obs_nudging") dest_dir = os.path.join(cfg.cosmo_input, "obs_nudging") tools.create_dir(dest_dir, "obs nudging input") @@ -56,3 +57,5 @@ def main(cfg): tools.copy_file(os.path.join(cfg.obs_nudging_dir, 'blklsttmp'), dest_dir) logging.info("Copied blacklist-file to {}".format(dest_dir)) + + cfg.finish_time_logging("obs_nudging", launch_time) \ No newline at end of file diff --git a/jobs/octe.py b/jobs/octe.py index f95193e1..a2043572 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -177,6 +177,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + launch_time = cfg.init_time_logging("octe") dest_dir = join(cfg.cosmo_input, 'octe') create_dir_and_copy_input(dest_dir=dest_dir, lambdas_src=cfg.octe_lambdas, @@ -187,3 +188,4 @@ def main(cfg): logging.info("Starting to create BG-ensembles in " + cfg.int2lm_output) perturb_bgs_in_dir(cfg.octe_lambdas, cfg.int2lm_output) logging.info("Finished creating BG-ensembles") + cfg.finish_time_logging("octe", launch_time) \ No newline at end of file diff --git a/jobs/oem.py b/jobs/oem.py index 4217b48e..44705594 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -22,6 +22,7 @@ def main(cfg): If an error occurs during the process. """ cosmo.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("oem") oem_dir = cfg.oem['dir'] oem_gridded_emissions_nc = os.path.join(oem_dir, @@ -82,3 +83,5 @@ def main(cfg): if cfg.oem.get('oem_ens_lambda_nc') is not None: tools.copy_file(os.path.join(oem_dir, cfg.oem['ens_lambda_nc']), os.path.join(dest_dir, cfg.oem['ens_lambda_nc'])) + + cfg.finish_time_logging("oem", launch_time) \ No newline at end of file diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 740b92b1..50316e74 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -16,6 +16,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + launch_time = cfg.init_time_logging("online_vprm") dest_modis = 'modis.nc' dest_vegetation = 'vegetation.nc' @@ -33,3 +34,4 @@ def main(cfg): src_dir, dest_dir)) tools.copy_file(vegetation_data_nc, os.path.join(dest_dir, dest_vegetation)) + cfg.finish_time_logging("online_vprm", launch_time) \ No newline at end of file diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index 72536394..f641d572 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -21,6 +21,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.check_model(cfg, 'cosmo-art') + launch_time = cfg.init_time_logging("photo_rate") logging.info("Copying photolysis-rate file from {} to {}".format( cfg.photo_rate_file, @@ -35,3 +36,5 @@ def main(cfg): tools.copy_file(src_file, dest_path) logging.info("Finished") + + cfg.finish_time_logging("photo_rate", launch_time) \ No newline at end of file diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 72dae422..f4929875 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -65,6 +65,7 @@ def main(cfg): """ int2lm.set_cfg_variables(cfg) cosmo.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("post_cosmo") logfile = os.path.join(cfg.log_working_dir, "post_cosmo") runscript_path = os.path.join(cfg.cosmo_run, "post_cosmo.job") @@ -138,3 +139,5 @@ def main(cfg): if exitcode != 0: raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + + cfg.finish_time_logging("post_cosmo", launch_time) \ No newline at end of file diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 129115b7..f8d51628 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -29,6 +29,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ int2lm.set_cfg_variables(cfg) + launch_time = cfg.init_time_logging("post_int2lm") # Int2lm processing always starts at hstart=0, thus modifying inidate inidate_int2lm_yyyymmddhh = cfg.startdate_sim_yyyymmddhh @@ -151,3 +152,5 @@ def main(cfg): =====================================================""" % date.strftime('%s') logging.info(to_print) + + cfg.finish_time_logging("post_int2lm", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index dad6a306..237b20af 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -7,7 +7,7 @@ import shutil import subprocess from . import tools, prepare_icon -from pathlib import Path +from pathlib import Path # noqa: F401 from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond # noqa: F401 from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index 8736df2b..55d6c594 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -42,6 +42,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes """ + launch_time = cfg.init_time_logging("reduce_output") cosmo_output = cfg.cosmo_output output_path = cfg.cosmo_output_reduced @@ -159,3 +160,5 @@ def main(cfg): =====================================================""" % date.strftime("%s") logging.info(to_print) + + cfg.finish_time_logging("reduce_output", launch_time) \ No newline at end of file diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 45aa4eab..4ae6ff82 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -38,6 +38,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes """ + launch_time = cfg.init_time_logging("verify_chain") logging.info("Started verification") for (ref_file, run_file), variables in cfg.verify_chain['values_to_check'].items(): @@ -67,3 +68,5 @@ def main(cfg): comp_data(ref_data, run_data, variables) logging.info("Finished verification") + + cfg.finish_time_logging("verify_chain", launch_time) \ No newline at end of file From 18d9fc35920a30363ae7dae0ca4789d3a3b3d8c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 15:00:52 +0100 Subject: [PATCH 112/245] Merge geosp into prepare_art --- cases/icon-seq-test/config.yaml | 1 - jobs/__init__.py | 1 - jobs/geosp.py | 61 ----------------- jobs/prepare_art.py | 112 ++++++++++++++++++++++---------- workflows.yaml | 19 +----- 5 files changed, 80 insertions(+), 114 deletions(-) delete mode 100644 jobs/geosp.py diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml index 341bb646..f1b520d1 100644 --- a/cases/icon-seq-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -6,7 +6,6 @@ workflow: jobs: - prepare_icon - icontools - - geosp - icon constraint: gpu run_on: cpu diff --git a/jobs/__init__.py b/jobs/__init__.py index 1934831f..332f34a8 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -5,7 +5,6 @@ from . import check_output from . import cosmo from . import emissions -from . import geosp from . import icon from . import icontools from . import int2lm diff --git a/jobs/geosp.py b/jobs/geosp.py deleted file mode 100644 index b15d2332..00000000 --- a/jobs/geosp.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -from pathlib import Path # noqa: F401 -import logging -import xarray as xr -import numpy as np -from . import tools, prepare_icon - - -def main(cfg): - """ - The constant variable 'GEOSP' is added to the files not containing it - using python-cdo bindings. - """ - prepare_icon.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("geosp") - logging.info("Add GEOSP to all meteo files") - - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - # Specify file names - geosp_filename = time.replace( - hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) - - # Copy GEOSP file from last run if not present - if not os.path.exists(geosp_file): - geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) - tools.copy_file(geosp_src_file, - cfg.icon_input_icbc, - output_log=True) - - # Load GEOSP data array as da_geosp at time 00: - ds = xr.open_dataset(src_file) - ds_geosp = xr.open_dataset(geosp_file) - da_geosp = ds_geosp['GEOSP'] - - # Merge GEOSP-dataset with other timesteps - if (time.hour != 0): - # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) - # Merge GEOSP into temporary file - ds_merged = xr.merge([ds, da_geosp]) - ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Logging info for merging GEOSP - logging.info("Added GEOSP to file {}".format(merged_file)) - # Rename file to get original file name - tools.rename_file(merged_file, src_file) - - cfg.finish_time_logging("geosp", launch_time) \ No newline at end of file diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index eac45644..b2611536 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -10,45 +10,89 @@ def main(cfg): """ - Add Q (copy of QV) and/or PS to initial file + Prepare ICON-ART simulations. + + - Add GEOSP to all meteo files that don't contain it + - Add Q (copy of QV) and/or PS to initial file """ prepare_icon.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") - logging.info('Add Q (copy of QV) and/or PS to initial file') - if cfg.art_input_folder: - meteo_file = os.path.join( + logging.info("Add GEOSP to all meteo files") + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + # Specify file names + geosp_filename = time.replace( + hour=0).strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) + src_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + src_file = os.path.join(cfg.icon_input_icbc, src_filename) + merged_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) + + # Copy GEOSP file from last run if not present + if not os.path.exists(geosp_file): + geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, + geosp_filename) + tools.copy_file(geosp_src_file, + cfg.icon_input_icbc, + output_log=True) + + # Load GEOSP data array as da_geosp at time 00: + ds = xr.open_dataset(src_file) + ds_geosp = xr.open_dataset(geosp_file) + da_geosp = ds_geosp['GEOSP'] + + # Merge GEOSP-dataset with other timesteps + if (time.hour != 0): + # Change values of time dimension to current time + da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) + # Merge GEOSP into temporary file + ds_merged = xr.merge([ds, da_geosp]) + ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Logging info for merging GEOSP + logging.info("Added GEOSP to file {}".format(merged_file)) + # Rename file to get original file name + tools.rename_file(merged_file, src_file) + logging.info('OK') + + logging.info('Add Q (copy of QV) and/or PS to initial file') + meteo_file = os.path.join( + cfg.icon_input_icbc, + cfg.startdate_sim.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '.nc') + if os.path.isfile(meteo_file): + merged_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') - if os.path.isfile(meteo_file): - merged_file = os.path.join( - cfg.icon_input_icbc, - cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') - ds = xr.open_dataset(meteo_file) - merging = False - if 'PS' not in ds: - if 'LNPS' not in ds: - raise KeyError( - f"'LNPS' must be found in the initial conditions file {meteo_file}" - ) - merging = True - ds['PS'] = ds['LNPS'] - ds['PS'].attrs = ds['LNPS'].attrs - ds['PS'] = np.exp(ds['PS']) - ds['PS'] = ds['PS'].squeeze(dim='lev_2') - ds['PS'].attrs["long_name"] = 'surface pressure' - ds['PS'].attrs['units'] = 'Pa' - logging.info(f"Added PS to file {meteo_file}") - if 'Q' not in ds: - merging = True - ds['Q'] = ds['QV'] - logging.info(f"Added Q to file {meteo_file}") - if merging: - ds.to_netcdf(merged_file) - tools.rename_file(merged_file, meteo_file) - + cfg.meteo['nameformat']) + + '_merged.nc') + ds = xr.open_dataset(meteo_file) + merging = False + if 'PS' not in ds: + if 'LNPS' not in ds: + raise KeyError( + f"'LNPS' must be found in the initial conditions file {meteo_file}" + ) + merging = True + ds['PS'] = ds['LNPS'] + ds['PS'].attrs = ds['LNPS'].attrs + ds['PS'] = np.exp(ds['PS']) + ds['PS'] = ds['PS'].squeeze(dim='lev_2') + ds['PS'].attrs["long_name"] = 'surface pressure' + ds['PS'].attrs['units'] = 'Pa' + logging.info(f"Added PS to file {meteo_file}") + if 'Q' not in ds: + merging = True + ds['Q'] = ds['QV'] + logging.info(f"Added Q to file {meteo_file}") + if merging: + ds.to_netcdf(merged_file) + tools.rename_file(merged_file, meteo_file) logging.info('OK') + cfg.finish_time_logging("prepare_art", launch_time) \ No newline at end of file diff --git a/workflows.yaml b/workflows.yaml index a7d8d56d..d2fd09dc 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -71,20 +71,15 @@ icon: jobs: - prepare_icon - icontools - - geosp - icon dependencies: icontools: current: - prepare_icon - geosp: - current: - - icontools icon: current: - prepare_icon - icontools - - geosp previous: - icon @@ -94,24 +89,19 @@ icon-art: jobs: - prepare_icon - icontools - - geosp - prepare_art - icon dependencies: icontools: current: - prepare_icon - geosp: - current: - - icontools prepare_art: current: - - geosp + - icontools icon: current: - prepare_icon - icontools - - geosp - prepare_art previous: - icon @@ -142,7 +132,6 @@ icon-art-oem: jobs: - prepare_icon - icontools - - geosp - prepare_art - prepare_art_oem - icon @@ -150,12 +139,9 @@ icon-art-oem: icontools: current: - prepare_icon - geosp: - current: - - icontools prepare_art: current: - - geosp + - icontools prepare_art_oem: current: - prepare_art @@ -163,7 +149,6 @@ icon-art-oem: current: - prepare_icon - icontools - - geosp - prepare_art - prepare_art_oem previous: From c36f70dd75fb43785987ccf5d285ff27bf5e695f Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 9 Jan 2024 14:01:25 +0000 Subject: [PATCH 113/245] GitHub Action: Apply Pep8-formatting --- jobs/int2lm.py | 3 ++- jobs/prepare_art.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 0293d123..9be38614 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -13,6 +13,7 @@ def set_cfg_variables(cfg): cfg.int2lm_run = cfg.chain_root / 'int2lm' / 'run' cfg.int2lm_output = cfg.chain_root / 'int2lm' / 'output' + def main(cfg): """Setup the namelist for int2lm and submit the job to the queue. @@ -165,4 +166,4 @@ def main(cfg): if exitcode != 0: raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) - cfg.finish_time_logging("int2lm", launch_time) \ No newline at end of file + cfg.finish_time_logging("int2lm", launch_time) diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index b2611536..0877d622 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -64,13 +64,12 @@ def main(cfg): meteo_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '.nc') + cfg.meteo['nameformat']) + '.nc') if os.path.isfile(meteo_file): merged_file = os.path.join( cfg.icon_input_icbc, cfg.startdate_sim.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + - '_merged.nc') + cfg.meteo['nameformat']) + '_merged.nc') ds = xr.open_dataset(meteo_file) merging = False if 'PS' not in ds: @@ -95,4 +94,4 @@ def main(cfg): tools.rename_file(merged_file, meteo_file) logging.info('OK') - cfg.finish_time_logging("prepare_art", launch_time) \ No newline at end of file + cfg.finish_time_logging("prepare_art", launch_time) From 729fb6081cf379ac7be1104fb181e3c3b525ecda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 15:11:33 +0100 Subject: [PATCH 114/245] Include oem in icon-art-oem workflow --- workflows.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workflows.yaml b/workflows.yaml index d2fd09dc..a86069e3 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -131,6 +131,7 @@ icon-art-oem: - restart jobs: - prepare_icon + - oem - icontools - prepare_art - prepare_art_oem @@ -144,10 +145,12 @@ icon-art-oem: - icontools prepare_art_oem: current: + - oem - prepare_art icon: current: - prepare_icon + - oem - icontools - prepare_art - prepare_art_oem From 32a557c3d91603a25bd76bada36db86f003b6fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 15:14:48 +0100 Subject: [PATCH 115/245] Fix icon job --- jobs/icon.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 3cb8d059..649faafa 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import logging -from . import tools, prepare_icon, prepare_art_global, prepare_art_oem +from . import tools, prepare_icon def main(cfg): @@ -32,8 +32,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_icon.set_cfg_variables(cfg) - prepare_art_global.set_cfg_variables(cfg) - prepare_art_oem.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("icon") logfile = cfg.log_working_dir / "icon" From f23c337112ebe732e478cbc6a2f2a3cc5eec8bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 9 Jan 2024 15:37:50 +0100 Subject: [PATCH 116/245] Move geosp to icontools job --- jobs/icontools.py | 48 ++++++++++++++++++++++++++++++++- jobs/prepare_icon.py | 61 +++++++++++++++++++++--------------------- jobs/tools/__init__.py | 1 + 3 files changed, 78 insertions(+), 32 deletions(-) diff --git a/jobs/icontools.py b/jobs/icontools.py index e93d0ddd..6dcd808f 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -2,11 +2,15 @@ # -*- coding: utf-8 -*- import logging +import os +import xarray as xr +import numpy as np from . import tools, prepare_icon def main(cfg): """ + - Add GEOSP to all meteo files - Submit the runscript for the DWD ICON tools to remap the meteorological files. - All runscripts specified in ``cfg.icontools_runjobs`` are submitted. - The meteorological files are read from the original input directory @@ -18,6 +22,48 @@ def main(cfg): logfile = cfg.log_working_dir / "icontools" logfile_finish = cfg.log_finished_dir / "icontools" + logging.info("Add GEOSP to all meteo files") + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + # Specify file names + geosp_filename = time.replace( + hour=0).strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) + src_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + src_file = os.path.join(cfg.icon_input_icbc, src_filename) + merged_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) + + # Copy GEOSP file from last run if not present + if not os.path.exists(geosp_file): + geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, + geosp_filename) + tools.copy_file(geosp_src_file, + cfg.icon_input_icbc, + output_log=True) + + # Load GEOSP data array as da_geosp at time 00: + ds = xr.open_dataset(src_file) + ds_geosp = xr.open_dataset(geosp_file) + da_geosp = ds_geosp['GEOSP'] + + # Merge GEOSP-dataset with other timesteps + if (time.hour != 0): + # Change values of time dimension to current time + da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) + # Merge GEOSP into temporary file + ds_merged = xr.merge([ds, da_geosp]) + ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Logging info for merging GEOSP + logging.info("Added GEOSP to file {}".format(merged_file)) + # Rename file to get original file name + tools.rename_file(merged_file, src_file) + logging.info('OK') + #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) #----------------------------------------------------- @@ -41,7 +87,7 @@ def main(cfg): datafile_list_chem = ' '.join([str(v) for v in datafile_list_chem]) #----------------------------------------------------- - # Write and submit runscripts + # Write and submit ICONTOOLS runscripts #----------------------------------------------------- dep_id = None for runscript in cfg.icontools_runjobs: diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 32b6d9ee..cf900522 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -69,36 +69,35 @@ def main(cfg): set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_icon") - if cfg.workflow_name.startswith('icon'): - logging.info('ICON input data (IC/BC)') - - # Create directories - tools.create_dir(cfg.icon_work, "icon_work") - tools.create_dir(cfg.icon_input_icbc, "icon_input_icbc") - tools.create_dir(cfg.icon_output, "icon_output") - tools.create_dir(cfg.icon_restart_out, "icon_restart_out") - - # Set logfile - logfile = cfg.log_working_dir / 'prepare_icon' - - # Copy input files to scratch - script_lines = [ - '#!/usr/bin/env bash', - f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', - f'#SBATCH --account={cfg.compute_account}', - '#SBATCH --time=00:10:00', - f'#SBATCH --partition={cfg.compute_queue}', - f'#SBATCH --constraint={cfg.constraint}', '#SBATCH --nodes=1', - f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', - f'#SBATCH --chdir={cfg.icon_work}', '' - ] - for target, destination in zip(cfg.input_files.values(), - cfg.input_files_scratch.values()): - script_lines.append(f'rsync -av {target} {destination}') - - with (script := cfg.icon_work / 'copy_input.job').open('w') as f: - f.write('\n'.join(script_lines)) - - cfg.submit('prepare_icon', script) + # Create directories + tools.create_dir(cfg.icon_work, "icon_work") + tools.create_dir(cfg.icon_input_icbc, "icon_input_icbc") + tools.create_dir(cfg.icon_output, "icon_output") + tools.create_dir(cfg.icon_restart_out, "icon_restart_out") + + # Set logfile + logfile = cfg.log_working_dir / 'prepare_icon' + + logging.info('Copy ICON input data (IC/BC) to working directory') + # Copy input files to scratch + script_lines = [ + '#!/usr/bin/env bash', + f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', + f'#SBATCH --account={cfg.compute_account}', + '#SBATCH --time=00:10:00', + f'#SBATCH --partition={cfg.compute_queue}', + f'#SBATCH --constraint={cfg.constraint}', '#SBATCH --nodes=1', + f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', + f'#SBATCH --chdir={cfg.icon_work}', '' + ] + for target, destination in zip(cfg.input_files.values(), + cfg.input_files_scratch.values()): + script_lines.append(f'rsync -av {target} {destination}') + + with (script := cfg.icon_work / 'copy_input.job').open('w') as f: + f.write('\n'.join(script_lines)) + + cfg.submit('prepare_icon', script) + logging.info("OK") cfg.finish_time_logging("prepare_icon", launch_time) diff --git a/jobs/tools/__init__.py b/jobs/tools/__init__.py index b21a6569..1abd2c08 100644 --- a/jobs/tools/__init__.py +++ b/jobs/tools/__init__.py @@ -195,6 +195,7 @@ def create_dir(path, readable_name): """ try: os.makedirs(path, exist_ok=True) + logging.info(f"Created {readable_name} directory at path {path}") except (OSError, Exception) as e: logging.error("Creating {} directory at path {} failed with {}".format( readable_name, path, From 83902f1ab01c562654f6d64712e2d208a8ac97c8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 9 Jan 2024 14:38:20 +0000 Subject: [PATCH 117/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_icon.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index cf900522..6f69e73c 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -83,15 +83,14 @@ def main(cfg): script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="copy_input_{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}"', - f'#SBATCH --account={cfg.compute_account}', - '#SBATCH --time=00:10:00', + f'#SBATCH --account={cfg.compute_account}', '#SBATCH --time=00:10:00', f'#SBATCH --partition={cfg.compute_queue}', f'#SBATCH --constraint={cfg.constraint}', '#SBATCH --nodes=1', f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', f'#SBATCH --chdir={cfg.icon_work}', '' ] for target, destination in zip(cfg.input_files.values(), - cfg.input_files_scratch.values()): + cfg.input_files_scratch.values()): script_lines.append(f'rsync -av {target} {destination}') with (script := cfg.icon_work / 'copy_input.job').open('w') as f: From bae1374c1a31f3b7484e11dd99d3d918ae6af505 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Tue, 9 Jan 2024 21:30:28 +0100 Subject: [PATCH 118/245] add: placeholder for monitoring async jobs --- run_chain.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/run_chain.py b/run_chain.py index b177ce1e..a51f4a35 100755 --- a/run_chain.py +++ b/run_chain.py @@ -247,6 +247,17 @@ def run_chunk(cfg, force, resume): # wait for previous chunk to be done cfg.wait_for_previous() + # - [ ] Matthieu : Monitor current chunk + # For each job check and print: + # - job id + # - dependencies + # - status (State) + # - elapsed time (Elapsed) + # - start time (Start) + # - end time (End) + # - partition (Partition) + # - number of nodes (NNodes) + # cycle cfg.job_ids['previous'] = cfg.job_ids['current'] else: From 035ace8902c1c31afce85c80d1e73b572e8e5f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 11:52:59 +0100 Subject: [PATCH 119/245] Run geosp after icontools --- jobs/icontools.py | 84 +++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/jobs/icontools.py b/jobs/icontools.py index 6dcd808f..69322c6e 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -22,48 +22,6 @@ def main(cfg): logfile = cfg.log_working_dir / "icontools" logfile_finish = cfg.log_finished_dir / "icontools" - logging.info("Add GEOSP to all meteo files") - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - # Specify file names - geosp_filename = time.replace( - hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) - - # Copy GEOSP file from last run if not present - if not os.path.exists(geosp_file): - geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) - tools.copy_file(geosp_src_file, - cfg.icon_input_icbc, - output_log=True) - - # Load GEOSP data array as da_geosp at time 00: - ds = xr.open_dataset(src_file) - ds_geosp = xr.open_dataset(geosp_file) - da_geosp = ds_geosp['GEOSP'] - - # Merge GEOSP-dataset with other timesteps - if (time.hour != 0): - # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) - # Merge GEOSP into temporary file - ds_merged = xr.merge([ds, da_geosp]) - ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Logging info for merging GEOSP - logging.info("Added GEOSP to file {}".format(merged_file)) - # Rename file to get original file name - tools.rename_file(merged_file, src_file) - logging.info('OK') - #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) #----------------------------------------------------- @@ -108,4 +66,46 @@ def main(cfg): logging.info(f" Starting icontools runscript {runscript}.") dep_id = cfg.submit('icontools', runscript_path, add_dep=dep_id) + logging.info("Add GEOSP to all meteo files") + for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, + cfg.meteo['inc']): + # Specify file names + geosp_filename = time.replace( + hour=0).strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) + src_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_lbc.nc' + src_file = os.path.join(cfg.icon_input_icbc, src_filename) + merged_filename = time.strftime(cfg.meteo['prefix'] + + cfg.meteo['nameformat']) + '_merged.nc' + merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) + + # Copy GEOSP file from last run if not present + if not os.path.exists(geosp_file): + geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, + geosp_filename) + tools.copy_file(geosp_src_file, + cfg.icon_input_icbc, + output_log=True) + + # Load GEOSP data array as da_geosp at time 00: + ds = xr.open_dataset(src_file) + ds_geosp = xr.open_dataset(geosp_file) + da_geosp = ds_geosp['GEOSP'] + + # Merge GEOSP-dataset with other timesteps + if (time.hour != 0): + # Change values of time dimension to current time + da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) + # Merge GEOSP into temporary file + ds_merged = xr.merge([ds, da_geosp]) + ds_merged.attrs = ds.attrs + ds_merged.to_netcdf(merged_file) + # Logging info for merging GEOSP + logging.info("Added GEOSP to file {}".format(merged_file)) + # Rename file to get original file name + tools.rename_file(merged_file, src_file) + logging.info('OK') + cfg.finish_time_logging("icontools", launch_time) From 814eb5ccfe491d421a953f80b16191d988081df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 12:19:27 +0100 Subject: [PATCH 120/245] Remove oem job from icon-art-oem case --- workflows.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/workflows.yaml b/workflows.yaml index a86069e3..d2fd09dc 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -131,7 +131,6 @@ icon-art-oem: - restart jobs: - prepare_icon - - oem - icontools - prepare_art - prepare_art_oem @@ -145,12 +144,10 @@ icon-art-oem: - icontools prepare_art_oem: current: - - oem - prepare_art icon: current: - prepare_icon - - oem - icontools - prepare_art - prepare_art_oem From 850334023430e5bf988ee175e3b29480e50cee73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 13:32:36 +0100 Subject: [PATCH 121/245] Change to current logfile within jobs --- config.py | 6 +++--- jobs/biofluxes.py | 1 + jobs/check_output.py | 13 ++++++------ jobs/cosmo.py | 9 ++++---- jobs/emissions.py | 1 + jobs/icon.py | 10 ++++----- jobs/icontools.py | 7 +++---- jobs/int2lm.py | 9 +++----- jobs/obs_nudging.py | 1 + jobs/octe.py | 1 + jobs/oem.py | 1 + jobs/online_vprm.py | 1 + jobs/photo_rate.py | 1 + jobs/post_cosmo.py | 6 +++--- jobs/post_int2lm.py | 1 + jobs/prepare_art.py | 42 -------------------------------------- jobs/prepare_art_global.py | 1 + jobs/prepare_art_oem.py | 1 + jobs/prepare_icon.py | 6 ++---- jobs/reduce_output.py | 4 ++-- jobs/verify_chain.py | 1 + run_chain.py | 14 ++++++------- 22 files changed, 47 insertions(+), 90 deletions(-) diff --git a/config.py b/config.py index 8bc9fc8f..ea030ca9 100644 --- a/config.py +++ b/config.py @@ -466,7 +466,7 @@ def submit(self, job_name, script, add_dep=None, logfile=None): self.job_ids['current'][job_name].append(job_id) exitcode = result.returncode - self.check_job(exitcode, logfile) + self.check_job(exitcode, logfile=logfile) return job_id @@ -482,7 +482,7 @@ def check_job(self, exitcode, logfile=None): if exitcode != 0: raise RuntimeError(f"sbatch returned exitcode {exitcode}") - def create_sbatch_script(self, job_name, log_file): + def create_sbatch_script(self, job_name): """Create an sbatch script to launch jobs individually. Use run_chain.py arguments to submit those jobs. """ @@ -490,7 +490,7 @@ def create_sbatch_script(self, job_name, log_file): '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}_{self.job_id}"', '#SBATCH --nodes=1', - f'#SBATCH --output={log_file}', + f'#SBATCH --output={self.logfile}', '#SBATCH --open-mode=append', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 7fc0628e..f400e351 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -19,6 +19,7 @@ def main(cfg): """ tools.check_model(cfg, 'cosmo-ghg') prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("biofluxes") scratch_path = os.path.join(cfg.int2lm_input, 'vprm') diff --git a/jobs/check_output.py b/jobs/check_output.py index 51e70178..035c9600 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -713,6 +713,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("check_output") date = dt.datetime.today() @@ -723,8 +724,6 @@ def main(cfg): ============== StartTime: %s =====================================================""" % date.strftime("%s") - logfile = os.path.join(cfg.log_working_dir, "check_output") - logging.basicConfig(filename=logfile, level=logging.INFO) logging.info(to_print) # if cfg.compute_host!="daint": @@ -749,7 +748,7 @@ def main(cfg): #SBATCH --time=00:30:00 #SBATCH --constraint=mc #SBATCH --ntasks=1 -#SBATCH --output={logfile} +#SBATCH --output={cfg.logfile} export EASYBUILD_PREFIX=/store/empa/em05/easybuild @@ -779,7 +778,7 @@ def main(cfg): cosmo_output=cfg.cosmo_output, output_root=cfg.output_root, work_log=cfg.log_working_dir, - logfile=logfile, + logfile=cfg.logfile, chain=cfg.chain_src_dir, chain_root=cfg.chain_root, action='get_data') @@ -820,7 +819,7 @@ def main(cfg): casename=cfg.casename, cosmo_output=cfg.cosmo_output, output_root=cfg.output_root, - logfile=logfile, + logfile=cfg.logfile, chain=cfg.chain_src_dir, chain_root=cfg.chain_root, action='plot_maps') @@ -860,9 +859,9 @@ def main(cfg): logging.info(to_print) # Check for errors - with open(logfile) as f: + with open(cfg.logfile) as f: if 'ERROR' in f.read(): - raise RuntimeError('Logfile containing errors! See %s' % logfile) + raise RuntimeError('Logfile containing errors! See %s' % cfg.logfile) cfg.finish_time_logging("check_output", launch_time) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 399d8f57..399c9b95 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -77,9 +77,8 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("cosmo") - logfile = os.path.join(cfg.log_working_dir, "cosmo") - logfile_finish = os.path.join(cfg.log_finished_dir, "cosmo") logging.info("Setup the namelist for a COSMO tracer run and " "submit the job to the queue") @@ -142,7 +141,7 @@ def main(cfg): laf_output_refdate=laf_output_refdate, )) # Execute fieldextra - with open(logfile, "a+") as log: + with open(cfg.logfile, "a+") as log: result = subprocess.run( [cfg.fieldextra_bin, output_file_merge], stdout=log) @@ -232,8 +231,8 @@ def main(cfg): cosmo_runscript.format(cfg=cfg, **cfg.cosmo, np_tot=np_tot, - logfile=logfile, - logfile_finish=logfile_finish)) + logfile=cfg.logfile, + logfile_finish=cfg.logfile_finish)) result = subprocess.run( ["sbatch", "--wait", diff --git a/jobs/emissions.py b/jobs/emissions.py index b3dd32bc..9f0efde0 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -31,6 +31,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("emissions") dest_prefix = "emis_" diff --git a/jobs/icon.py b/jobs/icon.py index 649faafa..9beda8d1 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -32,11 +32,9 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("icon") - logfile = cfg.log_working_dir / "icon" - logfile_finish = cfg.log_finished_dir / "icon" - logging.info("Setup the namelist for an ICON run and " "submit the job to the queue") @@ -60,12 +58,12 @@ def main(cfg): template = (cfg.case_path / cfg.icon_runjob_filename).read_text() script_str = template.format(cfg=cfg, inidata_filename=inidata_filename, - logfile=logfile, - logfile_finish=logfile_finish) + logfile=cfg.logfile, + logfile_finish=cfg.logfile_finish) script = (cfg.icon_work / 'run_icon.job') script.write_text(script_str) # Submit run script - cfg.submit('icon', script, logfile=logfile) + cfg.submit('icon', script, logfile=cfg.logfile) cfg.finish_time_logging("icon", launch_time) \ No newline at end of file diff --git a/jobs/icontools.py b/jobs/icontools.py index 69322c6e..2e86fb66 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -18,9 +18,8 @@ def main(cfg): in the input folder on scratch (``cfg.icon_input/icbc``). """ prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("icontools") - logfile = cfg.log_working_dir / "icontools" - logfile_finish = cfg.log_finished_dir / "icontools" #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) @@ -56,8 +55,8 @@ def main(cfg): outf.write( to_write.format(cfg=cfg, meteo=cfg.meteo, - logfile=logfile, - logfile_finish=logfile_finish, + logfile=cfg.logfile, + logfile_finish=cfg.logfile_finish, datafile_list=datafile_list, datafile_list_rest=datafile_list_rest, datafile_list_chem=datafile_list_chem)) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 9be38614..97578178 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -47,6 +47,7 @@ def main(cfg): """ prepare_cosmo.set_cfg_variables(cfg) set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("int2lm") # Total number of processes @@ -141,10 +142,6 @@ def main(cfg): cfg.int2lm['runjob_filename'])) as input_file: int2lm_runscript = input_file.read() - # Logfile variables - logfile = os.path.join(cfg.log_working_dir, "int2lm") - logfile_finish = os.path.join(cfg.log_finished_dir, "int2lm") - output_file = os.path.join(int2lm_run, "run.job") with open(output_file, "w") as outf: outf.write( @@ -155,8 +152,8 @@ def main(cfg): ini_hour=cfg.startdate_sim_yyyymmddhh[8:], np_tot=np_tot, hstop_int2lm=hstop_int2lm, - logfile=logfile, - logfile_finish=logfile_finish)) + logfile=cfg.logfile, + logfile_finish=cfg.logfile_finish)) # Submit job result = subprocess.run( diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index a199e42a..b1a9d353 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -27,6 +27,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("obs_nudging") dest_dir = os.path.join(cfg.cosmo_input, "obs_nudging") tools.create_dir(dest_dir, "obs nudging input") diff --git a/jobs/octe.py b/jobs/octe.py index a2043572..337eabb8 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -177,6 +177,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("octe") dest_dir = join(cfg.cosmo_input, 'octe') create_dir_and_copy_input(dest_dir=dest_dir, diff --git a/jobs/oem.py b/jobs/oem.py index 44705594..dbdf4cde 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -21,6 +21,7 @@ def main(cfg): RuntimeError If an error occurs during the process. """ + tools.change_logfile(cfg.logfile) cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("oem") diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 50316e74..c1444294 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -16,6 +16,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("online_vprm") dest_modis = 'modis.nc' dest_vegetation = 'vegetation.nc' diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index f641d572..3fb2a7da 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -20,6 +20,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + tools.change_logfile(cfg.logfile) tools.check_model(cfg, 'cosmo-art') launch_time = cfg.init_time_logging("photo_rate") diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index f4929875..aedcc267 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -65,9 +65,9 @@ def main(cfg): """ int2lm.set_cfg_variables(cfg) cosmo.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("post_cosmo") - logfile = os.path.join(cfg.log_working_dir, "post_cosmo") runscript_path = os.path.join(cfg.cosmo_run, "post_cosmo.job") copy_path = os.path.join( cfg.post_cosmo['output_root'], @@ -80,7 +80,7 @@ def main(cfg): runscript_content = "#!/bin/bash\n" runscript_content += runscript_header_template().format( compute_account=cfg.compute_account, - logfile=logfile, + logfile=cfg.logfile, constraint=cfg.constraint, cosmo_run=cfg.cosmo_run) @@ -132,7 +132,7 @@ def main(cfg): "ENDS", str(datetime.datetime.today()))) # copy own logfile aswell - tools.copy_file(logfile, os.path.join(copy_path, "logs/")) + tools.copy_file(cfg.logfile, os.path.join(copy_path, "logs/")) else: exitcode = call(["sbatch", runscript_path]) diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index f8d51628..42d173d1 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -29,6 +29,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ int2lm.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("post_int2lm") # Int2lm processing always starts at hstart=0, thus modifying inidate diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 0877d622..57cb9dc1 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -18,48 +18,6 @@ def main(cfg): prepare_icon.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("prepare_art") - logging.info("Add GEOSP to all meteo files") - for time in tools.iter_hours(cfg.startdate_sim, cfg.enddate_sim, - cfg.meteo['inc']): - # Specify file names - geosp_filename = time.replace( - hour=0).strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - geosp_file = os.path.join(cfg.icon_input_icbc, geosp_filename) - src_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_lbc.nc' - src_file = os.path.join(cfg.icon_input_icbc, src_filename) - merged_filename = time.strftime(cfg.meteo['prefix'] + - cfg.meteo['nameformat']) + '_merged.nc' - merged_file = os.path.join(cfg.icon_input_icbc, merged_filename) - - # Copy GEOSP file from last run if not present - if not os.path.exists(geosp_file): - geosp_src_file = os.path.join(cfg.icon_input_icbc_prev, - geosp_filename) - tools.copy_file(geosp_src_file, - cfg.icon_input_icbc, - output_log=True) - - # Load GEOSP data array as da_geosp at time 00: - ds = xr.open_dataset(src_file) - ds_geosp = xr.open_dataset(geosp_file) - da_geosp = ds_geosp['GEOSP'] - - # Merge GEOSP-dataset with other timesteps - if (time.hour != 0): - # Change values of time dimension to current time - da_geosp = da_geosp.assign_coords(time=[np.datetime64(time)]) - # Merge GEOSP into temporary file - ds_merged = xr.merge([ds, da_geosp]) - ds_merged.attrs = ds.attrs - ds_merged.to_netcdf(merged_file) - # Logging info for merging GEOSP - logging.info("Added GEOSP to file {}".format(merged_file)) - # Rename file to get original file name - tools.rename_file(merged_file, src_file) - logging.info('OK') - logging.info('Add Q (copy of QV) and/or PS to initial file') meteo_file = os.path.join( cfg.icon_input_icbc, diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 237b20af..8ab7da7f 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -22,6 +22,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("prepare_art_global") logging.info("Prepare ICON-ART for global simulations") diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 4d6be5f9..ee9874d6 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -12,6 +12,7 @@ def main(cfg): ICON-ART-OEM preparations """ prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("prepare_art") logging.info('Merging IC and LBC') diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 6f69e73c..2135db7e 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -67,6 +67,7 @@ def main(cfg): If any subprocess returns a non-zero exit code during execution. """ set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("prepare_icon") # Create directories @@ -75,9 +76,6 @@ def main(cfg): tools.create_dir(cfg.icon_output, "icon_output") tools.create_dir(cfg.icon_restart_out, "icon_restart_out") - # Set logfile - logfile = cfg.log_working_dir / 'prepare_icon' - logging.info('Copy ICON input data (IC/BC) to working directory') # Copy input files to scratch script_lines = [ @@ -86,7 +84,7 @@ def main(cfg): f'#SBATCH --account={cfg.compute_account}', '#SBATCH --time=00:10:00', f'#SBATCH --partition={cfg.compute_queue}', f'#SBATCH --constraint={cfg.constraint}', '#SBATCH --nodes=1', - f'#SBATCH --output={logfile}', '#SBATCH --open-mode=append', + f'#SBATCH --output={cfg.logfile}', '#SBATCH --open-mode=append', f'#SBATCH --chdir={cfg.icon_work}', '' ] for target, destination in zip(cfg.input_files.values(), diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index 55d6c594..6fac0eb0 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -42,6 +42,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes """ + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("reduce_output") cosmo_output = cfg.cosmo_output output_path = cfg.cosmo_output_reduced @@ -112,11 +113,10 @@ def main(cfg): py_file = os.path.join(tool_path, 'reduce_output_start_end.py') alternate_csv_file = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, 'variables.csv') - logfile = os.path.join(cfg.log_working_dir, 'reduce_output') logging.info('Submitting job to the queue...') result = subprocess.run([ - "sbatch", '--output=' + logfile, '--open-mode=append', '--wait', + "sbatch", '--output=' + cfg.logfile, '--open-mode=append', '--wait', bash_file, py_file, cosmo_output, output_path, str_startdate, str_enddate, str(cfg.reduce_output['output_levels']), diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 4ae6ff82..6b4e6c4a 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -38,6 +38,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes """ + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("verify_chain") logging.info("Started verification") for (ref_file, diff --git a/run_chain.py b/run_chain.py index a51f4a35..b3fc19b9 100755 --- a/run_chain.py +++ b/run_chain.py @@ -236,12 +236,11 @@ def run_chunk(cfg, force, resume): print(f' └── Process "{job}" for chunk "{cfg.job_id}"') # Logfile settings - logfile = cfg.log_working_dir / job - logfile_finish = cfg.log_finished_dir / job - tools.change_logfile(logfile) + cfg.logfile = cfg.log_working_dir / job + cfg.logfile_finish = cfg.log_finished_dir / job # Submit the job - script = cfg.create_sbatch_script(job, logfile) + script = cfg.create_sbatch_script(job) cfg.submit(job, script) # wait for previous chunk to be done @@ -298,15 +297,14 @@ def run_chunk(cfg, force, resume): try_count -= 1 try: # Change the log file - logfile = cfg.log_working_dir / job - logfile_finish = cfg.log_finished_dir / job - tools.change_logfile(logfile) + cfg.logfile = cfg.log_working_dir / job + cfg.logfile_finish = cfg.log_finished_dir / job # Launch the job to_call = getattr(jobs, job) to_call.main(cfg) - shutil.copy(logfile, logfile_finish) + shutil.copy(cfg.logfile, cfg.logfile_finish) exitcode = 0 try_count = 0 From 7a949cd3f2b0269bc7587a1057d428d2d459ac4e Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 10 Jan 2024 12:33:05 +0000 Subject: [PATCH 122/245] GitHub Action: Apply Pep8-formatting --- jobs/check_output.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/check_output.py b/jobs/check_output.py index 035c9600..c63c5fa5 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -861,7 +861,8 @@ def main(cfg): # Check for errors with open(cfg.logfile) as f: if 'ERROR' in f.read(): - raise RuntimeError('Logfile containing errors! See %s' % cfg.logfile) + raise RuntimeError('Logfile containing errors! See %s' % + cfg.logfile) cfg.finish_time_logging("check_output", launch_time) From 3ad7f3d823abc0f3f99fa4a7f5e2c0fc70a378d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 13:50:23 +0100 Subject: [PATCH 123/245] Add missing log inits --- jobs/prepare_art.py | 1 + jobs/prepare_cosmo.py | 1 + 2 files changed, 2 insertions(+) diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 57cb9dc1..c907fa8d 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -16,6 +16,7 @@ def main(cfg): - Add Q (copy of QV) and/or PS to initial file """ prepare_icon.set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("prepare_art") logging.info('Add Q (copy of QV) and/or PS to initial file') diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 0e26a3c0..ee1e054b 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -37,6 +37,7 @@ def main(cfg): If any subprocess returns a non-zero exit code during execution. """ set_cfg_variables(cfg) + tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("prepare_cosmo") logging.info('COSMO analysis data for IC/BC') From 4535b2b2a4d4dd6070528a8a941729f610b1c0fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 14:23:49 +0100 Subject: [PATCH 124/245] Configure root logger --- jobs/tools/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jobs/tools/__init__.py b/jobs/tools/__init__.py index 1abd2c08..ccc4cddd 100644 --- a/jobs/tools/__init__.py +++ b/jobs/tools/__init__.py @@ -166,14 +166,14 @@ def change_logfile(filename): """ fileh = logging.FileHandler(filename, 'a', delay=True) - # log_format = logging.Formatter('%(levelname)s:%(message)s') - # fileh.setFormatter(log_format) - log = logging.getLogger() # root logger if len(log.handlers) > 0: - log.handlers = [fileh] # set the new handler + # If there are existing handlers, replace them with the new handler + log.handlers = [fileh] else: - logging.basicConfig(filename=filename, level=logging.INFO) + # If no existing handlers, add the new handler + log.addHandler(fileh) + log.setLevel(logging.INFO) # Set the desired logging level def create_dir(path, readable_name): From 122525ad1766a5d7171f50ebfe7bb3ea876ebd1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 14:28:43 +0100 Subject: [PATCH 125/245] set logger --- jobs/tools/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/tools/__init__.py b/jobs/tools/__init__.py index ccc4cddd..2ca09a36 100644 --- a/jobs/tools/__init__.py +++ b/jobs/tools/__init__.py @@ -167,6 +167,7 @@ def change_logfile(filename): fileh = logging.FileHandler(filename, 'a', delay=True) + log = logging.getLogger() # root logger if len(log.handlers) > 0: # If there are existing handlers, replace them with the new handler log.handlers = [fileh] From 389b40aeb67e3b75bb20d4496e65f5dba6baaf6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 14:41:23 +0100 Subject: [PATCH 126/245] Fix logger --- jobs/tools/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jobs/tools/__init__.py b/jobs/tools/__init__.py index 2ca09a36..6aa0d401 100644 --- a/jobs/tools/__init__.py +++ b/jobs/tools/__init__.py @@ -166,15 +166,18 @@ def change_logfile(filename): """ fileh = logging.FileHandler(filename, 'a', delay=True) + log_format = logging.Formatter('%(levelname)s:%(message)s') + fileh.setFormatter(log_format) log = logging.getLogger() # root logger + log.setLevel(logging.INFO) # Set the desired logging level + if len(log.handlers) > 0: # If there are existing handlers, replace them with the new handler log.handlers = [fileh] else: # If no existing handlers, add the new handler log.addHandler(fileh) - log.setLevel(logging.INFO) # Set the desired logging level def create_dir(path, readable_name): From f49e3958d58197dd6263d1bbbad9aa3d57472cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 14:48:11 +0100 Subject: [PATCH 127/245] Format logging output --- jobs/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/tools/__init__.py b/jobs/tools/__init__.py index 6aa0d401..15f39ead 100644 --- a/jobs/tools/__init__.py +++ b/jobs/tools/__init__.py @@ -166,7 +166,7 @@ def change_logfile(filename): """ fileh = logging.FileHandler(filename, 'a', delay=True) - log_format = logging.Formatter('%(levelname)s:%(message)s') + log_format = logging.Formatter('%(levelname)s: %(message)s') fileh.setFormatter(log_format) log = logging.getLogger() # root logger From c257556ff77f43dbe7ab56d66e8ee2ea91568336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 10 Jan 2024 20:47:39 +0100 Subject: [PATCH 128/245] Introduce BASIC_PYTHON_JOB option to call jobs directly in async mode --- jobs/icon.py | 11 +++++++++-- run_chain.py | 20 +++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 9beda8d1..10662d30 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -4,6 +4,10 @@ import logging from . import tools, prepare_icon +BASIC_PYTHON_JOB = True +# FIXME: BASIC_PYTHON_JOB should be False, but with ICON-ART the sbatch job is +# always failing because of the "invalid pointer" error. This is caught +# if this module itself is submitted as an sbatch job. def main(cfg): """Setup the namelists for an ICON tracer run and submit the job to @@ -33,13 +37,15 @@ def main(cfg): """ prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("icon") + if BASIC_PYTHON_JOB: + launch_time = cfg.init_time_logging("icon") logging.info("Setup the namelist for an ICON run and " "submit the job to the queue") # Copy icon executable execname = 'icon.exe' + tools.create_dir(cfg.icon_work, "icon_work") tools.copy_file(cfg.icon_binary_file, cfg.icon_work / execname) # Symlink the restart file to the last run into the icon/run folder @@ -66,4 +72,5 @@ def main(cfg): # Submit run script cfg.submit('icon', script, logfile=cfg.logfile) - cfg.finish_time_logging("icon", launch_time) \ No newline at end of file + if BASIC_PYTHON_JOB: + cfg.finish_time_logging("icon", launch_time) \ No newline at end of file diff --git a/run_chain.py b/run_chain.py index b3fc19b9..d22ef3c7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -239,9 +239,23 @@ def run_chunk(cfg, force, resume): cfg.logfile = cfg.log_working_dir / job cfg.logfile_finish = cfg.log_finished_dir / job - # Submit the job - script = cfg.create_sbatch_script(job) - cfg.submit(job, script) + # Check for the global module variable 'BASIC_PYTHON_JOB'. + # If it doesn't exist, assume that the job has to be + # submitted via sbatch. Otherwise take the actual value. + if not hasattr(getattr(jobs, job), 'BASIC_PYTHON_JOB') or \ + getattr(jobs, job).BASIC_PYTHON_JOB: + # Submit the job via sbatch + script = cfg.create_sbatch_script(job) + cfg.submit(job, script) + else: + # Execute the job directly + # FIXME: The time logging doesn't represent the timings from + # the submitted job. Thus it is disabled there, e.g. + # in `icon.py`. One solution could be to use the new + # get_job_info() function to read the start/end/elapsed + # time from those jobs and write them into the + # `chain_status.log` file once that job has finished. + getattr(jobs, job).main(cfg) # wait for previous chunk to be done cfg.wait_for_previous() From 91e72f72535fd7e5d2f201f9add1f11fe2ff7c77 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 10 Jan 2024 19:48:09 +0000 Subject: [PATCH 129/245] GitHub Action: Apply Pep8-formatting --- jobs/icon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/icon.py b/jobs/icon.py index 10662d30..4bc34661 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -9,6 +9,7 @@ # always failing because of the "invalid pointer" error. This is caught # if this module itself is submitted as an sbatch job. + def main(cfg): """Setup the namelists for an ICON tracer run and submit the job to the queue. @@ -73,4 +74,4 @@ def main(cfg): cfg.submit('icon', script, logfile=cfg.logfile) if BASIC_PYTHON_JOB: - cfg.finish_time_logging("icon", launch_time) \ No newline at end of file + cfg.finish_time_logging("icon", launch_time) From 870289a1f2ac627fbe0d658faecf04a8912e9574 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Thu, 11 Jan 2024 09:44:14 +0100 Subject: [PATCH 130/245] add: only submit basic python jobs through a nested run_chain --- jobs/biofluxes.py | 5 ++++- jobs/check_output.py | 3 +++ jobs/emissions.py | 5 ++++- jobs/obs_nudging.py | 5 ++++- jobs/octe.py | 5 ++++- jobs/oem.py | 5 ++++- jobs/online_vprm.py | 5 ++++- jobs/photo_rate.py | 5 ++++- jobs/post_cosmo.py | 5 ++++- jobs/post_int2lm.py | 5 ++++- jobs/prepare_art.py | 3 +++ jobs/prepare_art_global.py | 3 +++ jobs/prepare_art_oem.py | 3 +++ jobs/prepare_cosmo.py | 3 +++ jobs/reduce_output.py | 5 ++++- jobs/verify_chain.py | 5 ++++- run_chain.py | 7 +++++-- 17 files changed, 64 insertions(+), 13 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 7fc0628e..ee4d6b7a 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -6,6 +6,9 @@ from . import tools, prepare_icon +BASIC_PYTHON_JOB = True + + def main(cfg): """Prepare biofluxes files for COSMO simulations. @@ -45,4 +48,4 @@ def main(cfg): logging.error( "Splitting or copying of GPP or/and RA files to scratch failed." ) - cfg.finish_time_logging("biofluxes", launch_time) \ No newline at end of file + cfg.finish_time_logging("biofluxes", launch_time) diff --git a/jobs/check_output.py b/jobs/check_output.py index 51e70178..2a620f92 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -29,6 +29,9 @@ import tools +BASIC_PYTHON_JOB = True + + def pkl_path(folder, pid=None): """ Returns the path (and creates it, if necessary) to the stored pandas data file. diff --git a/jobs/emissions.py b/jobs/emissions.py index b3dd32bc..83b3a3fe 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -7,6 +7,9 @@ from . import tools, prepare_icon +BASIC_PYTHON_JOB = True + + def main(cfg): """Copy emission files to the int2lm input directory. @@ -69,4 +72,4 @@ def main(cfg): if cfg.workflow_name.startswith('cosmo'): tools.string2char.main(dest_path) - cfg.finish_time_logging("emissions", launch_time) \ No newline at end of file + cfg.finish_time_logging("emissions", launch_time) diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index a199e42a..d9bd7990 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -9,6 +9,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def main(cfg): """Copy and rename the obs_nudging files to the **COSMO** input directory. @@ -58,4 +61,4 @@ def main(cfg): logging.info("Copied blacklist-file to {}".format(dest_dir)) - cfg.finish_time_logging("obs_nudging", launch_time) \ No newline at end of file + cfg.finish_time_logging("obs_nudging", launch_time) diff --git a/jobs/octe.py b/jobs/octe.py index a2043572..0678a43c 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -12,6 +12,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def create_dir_and_copy_input(dest_dir, lambdas_src, maps_src): """Create a directory at dest_dir (**COSMO** input) and copy src there. @@ -188,4 +191,4 @@ def main(cfg): logging.info("Starting to create BG-ensembles in " + cfg.int2lm_output) perturb_bgs_in_dir(cfg.octe_lambdas, cfg.int2lm_output) logging.info("Finished creating BG-ensembles") - cfg.finish_time_logging("octe", launch_time) \ No newline at end of file + cfg.finish_time_logging("octe", launch_time) diff --git a/jobs/oem.py b/jobs/oem.py index 44705594..6065e6eb 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -7,6 +7,9 @@ from . import tools, cosmo +BASIC_PYTHON_JOB = True + + def main(cfg): """Copy emission and profile files to the **cosmo** or **icon** input directory. @@ -84,4 +87,4 @@ def main(cfg): tools.copy_file(os.path.join(oem_dir, cfg.oem['ens_lambda_nc']), os.path.join(dest_dir, cfg.oem['ens_lambda_nc'])) - cfg.finish_time_logging("oem", launch_time) \ No newline at end of file + cfg.finish_time_logging("oem", launch_time) diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 50316e74..0eaeda23 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -7,6 +7,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def main(cfg): """Copy MODIS surface reflectance data and vegetation class fraction file to the **cosmo** input directory. @@ -34,4 +37,4 @@ def main(cfg): src_dir, dest_dir)) tools.copy_file(vegetation_data_nc, os.path.join(dest_dir, dest_vegetation)) - cfg.finish_time_logging("online_vprm", launch_time) \ No newline at end of file + cfg.finish_time_logging("online_vprm", launch_time) diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index f641d572..a99e5582 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -7,6 +7,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def main(cfg): """Copy photolysis-rate file to the **COSMOART** input directory. @@ -37,4 +40,4 @@ def main(cfg): logging.info("Finished") - cfg.finish_time_logging("photo_rate", launch_time) \ No newline at end of file + cfg.finish_time_logging("photo_rate", launch_time) diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index f4929875..0a1e9ee5 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -9,6 +9,9 @@ from . import tools, int2lm, cosmo +BASIC_PYTHON_JOB = True + + def logfile_header_template(): """Returns a template for the logfile-header""" return ("\n=====================================================\n" @@ -140,4 +143,4 @@ def main(cfg): if exitcode != 0: raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) - cfg.finish_time_logging("post_cosmo", launch_time) \ No newline at end of file + cfg.finish_time_logging("post_cosmo", launch_time) diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index f8d51628..d6b62843 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -9,6 +9,9 @@ from . import tools, int2lm +BASIC_PYTHON_JOB = True + + def main(cfg): """Combine multiple **int2lm** tracer-output files into a single one for **COSMO**. @@ -153,4 +156,4 @@ def main(cfg): logging.info(to_print) - cfg.finish_time_logging("post_int2lm", launch_time) \ No newline at end of file + cfg.finish_time_logging("post_int2lm", launch_time) diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 0877d622..37d6526d 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -8,6 +8,9 @@ from . import tools, prepare_icon +BASIC_PYTHON_JOB = True + + def main(cfg): """ Prepare ICON-ART simulations. diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 237b20af..db90fd25 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -12,6 +12,9 @@ from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging +BASIC_PYTHON_JOB = True + + def main(cfg): """ Prepare global ICON-ART simulations. diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 4d6be5f9..0c18ea80 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -7,6 +7,9 @@ from . import tools, prepare_icon +BASIC_PYTHON_JOB = True + + def main(cfg): """ ICON-ART-OEM preparations diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 0e26a3c0..3db42978 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -7,6 +7,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def set_cfg_variables(cfg): cfg.int2lm_root = cfg.chain_root / 'int2lm' cfg.int2lm_input = cfg.int2lm_root / 'input' diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index 55d6c594..f61fa23d 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -16,6 +16,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def main(cfg): """ Calculates 2D column data and writes them into a new netCDF file. @@ -161,4 +164,4 @@ def main(cfg): logging.info(to_print) - cfg.finish_time_logging("reduce_output", launch_time) \ No newline at end of file + cfg.finish_time_logging("reduce_output", launch_time) diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 4ae6ff82..e1d5ea02 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -8,6 +8,9 @@ from . import tools +BASIC_PYTHON_JOB = True + + def comp_data(dataset1, dataset2, variables): """Use tools.helper.datasets_equal to compare the datasets. """ @@ -69,4 +72,4 @@ def main(cfg): logging.info("Finished verification") - cfg.finish_time_logging("verify_chain", launch_time) \ No newline at end of file + cfg.finish_time_logging("verify_chain", launch_time) diff --git a/run_chain.py b/run_chain.py index a51f4a35..1a68813b 100755 --- a/run_chain.py +++ b/run_chain.py @@ -241,8 +241,11 @@ def run_chunk(cfg, force, resume): tools.change_logfile(logfile) # Submit the job - script = cfg.create_sbatch_script(job, logfile) - cfg.submit(job, script) + if hasattr(job, 'BASIC_PYTHON_JOB') and job.BASIC_PYTHON_JOB: + script = cfg.create_sbatch_script(job, logfile) + cfg.submit(job, script) + else: + job.main() # wait for previous chunk to be done cfg.wait_for_previous() From 208e9e0f3c502969aa58252608ef83f8bec7182d Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 11 Jan 2024 08:48:50 +0000 Subject: [PATCH 131/245] GitHub Action: Apply Pep8-formatting --- jobs/biofluxes.py | 1 - jobs/check_output.py | 1 - jobs/emissions.py | 1 - jobs/obs_nudging.py | 1 - jobs/octe.py | 1 - jobs/oem.py | 1 - jobs/online_vprm.py | 1 - jobs/photo_rate.py | 1 - jobs/post_cosmo.py | 1 - jobs/post_int2lm.py | 1 - jobs/prepare_art.py | 1 - jobs/prepare_art_global.py | 1 - jobs/prepare_art_oem.py | 1 - jobs/prepare_cosmo.py | 1 - jobs/reduce_output.py | 1 - jobs/verify_chain.py | 1 - 16 files changed, 16 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 135a5804..4bbfbc60 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -5,7 +5,6 @@ import logging from . import tools, prepare_icon - BASIC_PYTHON_JOB = True diff --git a/jobs/check_output.py b/jobs/check_output.py index 96b45cc8..22efd12e 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -28,7 +28,6 @@ except ImportError: import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/emissions.py b/jobs/emissions.py index ba98e167..f99e8fd1 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -6,7 +6,6 @@ from . import tools, prepare_icon - BASIC_PYTHON_JOB = True diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index fe52ab6a..1bd7c98a 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -8,7 +8,6 @@ from datetime import timedelta from . import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/octe.py b/jobs/octe.py index fa981836..058853d4 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -11,7 +11,6 @@ from . import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/oem.py b/jobs/oem.py index d9c44b71..9ff1eb62 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -6,7 +6,6 @@ from . import tools, cosmo - BASIC_PYTHON_JOB = True diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 81817fe7..a6488549 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -6,7 +6,6 @@ from . import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index 182f49f9..218dcaf7 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -6,7 +6,6 @@ from . import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 19f10e41..533f8f54 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -8,7 +8,6 @@ from . import tools, int2lm, cosmo - BASIC_PYTHON_JOB = True diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 1af8ce98..5823b423 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -8,7 +8,6 @@ from datetime import datetime, timedelta from . import tools, int2lm - BASIC_PYTHON_JOB = True diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 70fe920a..0f120fc2 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -7,7 +7,6 @@ import numpy as np from . import tools, prepare_icon - BASIC_PYTHON_JOB = True diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index d0bcdd0b..9ceac550 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -11,7 +11,6 @@ from .tools.interpolate_data import create_oh_for_restart, create_oh_for_inicond # noqa: F401 from .tools.fetch_external_data import fetch_era5, fetch_era5_nudging - BASIC_PYTHON_JOB = True diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 7e24398e..5432efb6 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -6,7 +6,6 @@ import xarray as xr from . import tools, prepare_icon - BASIC_PYTHON_JOB = True diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 19d6fc4b..f339a366 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -6,7 +6,6 @@ from datetime import timedelta from . import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index 15eb0842..b62a97a1 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -15,7 +15,6 @@ from . import tools - BASIC_PYTHON_JOB = True diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 0b1756fb..95f6a712 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -7,7 +7,6 @@ from . import tools - BASIC_PYTHON_JOB = True From 623e4355e64cefdc352127b0f1f6fa57a57c454c Mon Sep 17 00:00:00 2001 From: Matthieu Date: Thu, 11 Jan 2024 10:01:04 +0100 Subject: [PATCH 132/245] fix: loop over jobs in run_chunk --- run_chain.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/run_chain.py b/run_chain.py index d0d680a6..5736678b 100755 --- a/run_chain.py +++ b/run_chain.py @@ -227,24 +227,25 @@ def run_chunk(cfg, force, resume): cfg.job_ids['current'] = {} # Submit current chunk - for job in cfg.jobs: - if (cfg.log_finished_dir / job).exists() and not force: + for job_name in cfg.jobs: + if (cfg.log_finished_dir / job_name).exists() and not force: # Skip job if already finished - print(f' └── Skip "{job}" for chunk "{cfg.job_id}"') + print(f' └── Skip "{job_name}" for chunk "{cfg.job_id}"') skip = True else: - print(f' └── Process "{job}" for chunk "{cfg.job_id}"') + print(f' └── Process "{job_name}" for chunk "{cfg.job_id}"') # Logfile settings - cfg.logfile = cfg.log_working_dir / job - cfg.logfile_finish = cfg.log_finished_dir / job + cfg.logfile = cfg.log_working_dir / job_name + cfg.logfile_finish = cfg.log_finished_dir / job_name # Submit the job + job = getattr(jobs, job_name) if hasattr(job, 'BASIC_PYTHON_JOB') and job.BASIC_PYTHON_JOB: - script = cfg.create_sbatch_script(job, logfile) - cfg.submit(job, script) + script = cfg.create_sbatch_script(job_name, logfile) + cfg.submit(job_name, script) else: - job.main() + job.main(cfg) # wait for previous chunk to be done cfg.wait_for_previous() From b39e27d17947d10199a2133aece189f51c7c1cfb Mon Sep 17 00:00:00 2001 From: Matthieu Date: Thu, 11 Jan 2024 10:38:03 +0100 Subject: [PATCH 133/245] ref: job_id becomes chunk_id reflects reality and avoids confusion with actual jobs id --- config.py | 14 +++++++------- run_chain.py | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/config.py b/config.py index ea030ca9..2102e1d7 100644 --- a/config.py +++ b/config.py @@ -354,22 +354,22 @@ def log_job_status(self, job, status, launch_time, duration=None): with open(log_file, 'w') as f: f.write(header) - # Format duration and job_id + # Format duration and chunk_id if job == 'chain': if duration is not None: duration = self.format_duration(duration) - job_id = self.casename + chunk_id = self.casename else: if duration is not None: duration = f"{str(int(duration.total_seconds()))} s" - job_id = self.job_id + chunk_id = self.chunk_id # Log the job information launch_time = launch_time.strftime("%a %b %d %Y %H:%M:%S") if status == 'FINISH' and duration: - log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<24} {duration}\n" + log_entry = f"{job:<15} {chunk_id:<21} {status:<6} {launch_time:<24} {duration}\n" else: - log_entry = f"{job:<15} {job_id:<21} {status:<6} {launch_time:<24}\n" + log_entry = f"{job:<15} {chunk_id:<21} {status:<6} {launch_time:<24}\n" with open(log_file, 'a') as f: f.write(log_entry) @@ -488,7 +488,7 @@ def create_sbatch_script(self, job_name): """ script_lines = [ '#!/usr/bin/env bash', - f'#SBATCH --job-name="{job_name}_{self.job_id}"', + f'#SBATCH --job-name="{job_name}_{self.chunk_id}"', '#SBATCH --nodes=1', f'#SBATCH --output={self.logfile}', '#SBATCH --open-mode=append', @@ -499,7 +499,7 @@ def create_sbatch_script(self, job_name): f'cd {self.chain_src_dir}', 'eval "$(conda shell.bash hook)"', 'conda activate proc-chain', - f'./run_chain.py {self.casename} -j {job_name} -c {self.job_id} -f -s --no-logging', + f'./run_chain.py {self.casename} -j {job_name} -c {self.chunk_id} -f -s --no-logging', '', ] diff --git a/run_chain.py b/run_chain.py index 5736678b..23f3d5d3 100755 --- a/run_chain.py +++ b/run_chain.py @@ -145,8 +145,8 @@ def run_chunk(cfg, force, resume): cfg.enddate_sim_yyyymmddhh = cfg.enddate_sim.strftime('%Y%m%d%H') # Folder naming and structure - cfg.job_id = f'{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}' - cfg.chain_root = cfg.work_root / cfg.casename / cfg.job_id + cfg.chunk_id = f'{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}' + cfg.chain_root = cfg.work_root / cfg.casename / cfg.chunk_id # Config variables for spinup runs (datetimes, job-id, etc.) if hasattr(cfg, 'spinup'): @@ -162,14 +162,14 @@ def run_chunk(cfg, force, resume): if cfg.second_one: startdate_sim_yyyymmddhh_prev = (cfg.enddate_sim - timedelta( hours=2 * cfg.restart_step_hours)).strftime('%Y%m%d%H') - else: # all other runs (i.e., get job_id from previous run) + else: # all other runs (i.e., get chunk_id from previous run) startdate_sim_yyyymmddhh_prev = ( cfg.enddate_sim - timedelta(hours=2 * cfg.restart_step_hours + cfg.spinup)).strftime('%Y%m%d%H') - cfg.job_id_prev = f'{startdate_sim_yyyymmddhh_prev}_{enddate_sim_yyyymmddhh_prev}' - cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.job_id_prev + cfg.chunk_id_prev = f'{startdate_sim_yyyymmddhh_prev}_{enddate_sim_yyyymmddhh_prev}' + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' # No restart for spinup simulations (= default values for no restart) @@ -185,8 +185,8 @@ def run_chunk(cfg, force, resume): cfg.enddate_sim_prev_yyyymmddhh = cfg.enddate_sim_prev.strftime( '%Y%m%d%H') - cfg.job_id_prev = f'{cfg.startdate_sim_prev_yyyymmddhh}_{cfg.enddate_sim_prev_yyyymmddhh}' - cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.job_id_prev + cfg.chunk_id_prev = f'{cfg.startdate_sim_prev_yyyymmddhh}_{cfg.enddate_sim_prev_yyyymmddhh}' + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev # Set restart directories cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' @@ -203,7 +203,7 @@ def run_chunk(cfg, force, resume): # if ifs_hres_dir doesn't point to a directory, # it is the name of the mother run mother_name = cfg.meteo.dir - cfg.meteo.dir = cfg.work_root / mother_name / cfg.job_id / 'cosmo' / 'output' + cfg.meteo.dir = cfg.work_root / mother_name / cfg.chunk_id / 'cosmo' / 'output' cfg.meteo.inc = 1 cfg.meteo.prefix = 'lffd' @@ -230,10 +230,10 @@ def run_chunk(cfg, force, resume): for job_name in cfg.jobs: if (cfg.log_finished_dir / job_name).exists() and not force: # Skip job if already finished - print(f' └── Skip "{job_name}" for chunk "{cfg.job_id}"') + print(f' └── Skip "{job_name}" for chunk "{cfg.chunk_id}"') skip = True else: - print(f' └── Process "{job_name}" for chunk "{cfg.job_id}"') + print(f' └── Process "{job_name}" for chunk "{cfg.chunk_id}"') # Logfile settings cfg.logfile = cfg.log_working_dir / job_name @@ -274,7 +274,7 @@ def run_chunk(cfg, force, resume): while True: if (cfg.log_finished_dir / job).exists(): print( - f' └── Skip "{job}" for chunk "{cfg.job_id}"' + f' └── Skip "{job}" for chunk "{cfg.chunk_id}"' ) skip = True break @@ -283,7 +283,7 @@ def run_chunk(cfg, force, resume): break else: print( - f" └── Wait for {job} of chunk {cfg.job_id}" + f" └── Wait for {job} of chunk {cfg.chunk_id}" ) sys.stdout.flush() for _ in range(3000): @@ -314,7 +314,7 @@ def run_chunk(cfg, force, resume): try_count = 0 except Exception: subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( - job, cfg.job_id) + job, cfg.chunk_id) logging.exception(subject) if cfg.user_mail: message = tools.prepare_message( @@ -327,7 +327,7 @@ def run_chunk(cfg, force, resume): if exitcode != 0 or not (cfg.log_finished_dir / job).exists(): subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( - job, cfg.job_id) + job, cfg.chunk_id) if cfg.user_mail: message = tools.prepare_message(cfg.log_working_dir / job) @@ -369,8 +369,8 @@ def restart_runs(cfg, force, resume): continue cfg.chunks.append(job_id) - for job_id in cfg.chunks: - cfg.job_id = job_id + for chunk_id in cfg.chunks: + cfg.chunk_id = chunk_id cfg.startdate_sim_yyyymmddhh = job_id[0:10] cfg.enddate_sim_yyyymmddhh = job_id[-10:] cfg.startdate_sim = datetime.strptime( From 8962742ee69932c1210f7b98ab4e561752b59be3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 11 Jan 2024 09:50:43 +0000 Subject: [PATCH 134/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 23f3d5d3..82766639 100755 --- a/run_chain.py +++ b/run_chain.py @@ -233,7 +233,8 @@ def run_chunk(cfg, force, resume): print(f' └── Skip "{job_name}" for chunk "{cfg.chunk_id}"') skip = True else: - print(f' └── Process "{job_name}" for chunk "{cfg.chunk_id}"') + print( + f' └── Process "{job_name}" for chunk "{cfg.chunk_id}"') # Logfile settings cfg.logfile = cfg.log_working_dir / job_name From 1c56a68955a75c670d401564bcb31c83cbba2bc9 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Thu, 11 Jan 2024 10:55:06 +0100 Subject: [PATCH 135/245] fix: leftover `job_id` -> `chunk_id` --- jobs/cosmo.py | 4 ++-- run_chain.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 399c9b95..c3dae4d8 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -103,7 +103,7 @@ def main(cfg): hours=cfg.restart_step) work_root = os.path.dirname(os.path.dirname(cfg.chain_root)) last_output_path = os.path.join(work_root, cfg.casename, - cfg.job_id_prev, 'cosmo', 'output') + cfg.chunk_id_prev, 'cosmo', 'output') laf_output_refdate = cfg.startdate_sim.strftime("%Y%m%d%H") last_laf_filename = "laf" + laf_output_refdate # At the beginning, use original laf_startfile @@ -241,4 +241,4 @@ def main(cfg): if exitcode != 0: raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) - cfg.finish_time_logging("cosmo", launch_time) \ No newline at end of file + cfg.finish_time_logging("cosmo", launch_time) diff --git a/run_chain.py b/run_chain.py index 23f3d5d3..bf4fe9ee 100755 --- a/run_chain.py +++ b/run_chain.py @@ -293,7 +293,7 @@ def run_chunk(cfg, force, resume): (cfg.log_finished_dir / job).unlink(missing_ok=True) if not skip: - print(f' └── Process "{job}" for chunk "{cfg.job_id}"') + print(f' └── Process "{job}" for chunk "{cfg.chunk_id}"') sys.stdout.flush() try_count = 1 + (cfg.ntry - 1) * (job == 'cosmo') @@ -364,15 +364,15 @@ def restart_runs(cfg, force, resume): hours=cfg.restart_step_hours) startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") - job_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" + chunk_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" if enddate_sim > cfg.enddate: continue - cfg.chunks.append(job_id) + cfg.chunks.append(chunk_id) for chunk_id in cfg.chunks: cfg.chunk_id = chunk_id - cfg.startdate_sim_yyyymmddhh = job_id[0:10] - cfg.enddate_sim_yyyymmddhh = job_id[-10:] + cfg.startdate_sim_yyyymmddhh = chunk_id[0:10] + cfg.enddate_sim_yyyymmddhh = chunk_id[-10:] cfg.startdate_sim = datetime.strptime( cfg.startdate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) cfg.enddate_sim = datetime.strptime( From da105eb016d12a1d81e03f085a33ad5e699812e9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 11 Jan 2024 09:56:15 +0000 Subject: [PATCH 136/245] GitHub Action: Apply Pep8-formatting --- jobs/cosmo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index c3dae4d8..1bc70132 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -103,7 +103,8 @@ def main(cfg): hours=cfg.restart_step) work_root = os.path.dirname(os.path.dirname(cfg.chain_root)) last_output_path = os.path.join(work_root, cfg.casename, - cfg.chunk_id_prev, 'cosmo', 'output') + cfg.chunk_id_prev, 'cosmo', + 'output') laf_output_refdate = cfg.startdate_sim.strftime("%Y%m%d%H") last_laf_filename = "laf" + laf_output_refdate # At the beginning, use original laf_startfile From f853d98232a7991886603449fe30a491faa08400 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Thu, 11 Jan 2024 16:04:36 +0100 Subject: [PATCH 137/245] add(untested): Slurm monitoring --- config.py | 82 +++++++++++++++++++++++++++++++++++++++++++--------- run_chain.py | 19 +++++------- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/config.py b/config.py index 2102e1d7..ff13a850 100644 --- a/config.py +++ b/config.py @@ -9,6 +9,10 @@ class Config(): + # Requested slurm info keys and corresponding printed width + info_requests = {'JobName': 10, 'JobID': 8, 'Partition': 9, 'NNodes': 6, + 'State': 14, 'Start': 13, 'End': 13, 'Elapsed': 9} + def __init__(self, casename): """Initialize an instance of the Config class. @@ -536,22 +540,74 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) - def get_job_info(jobid, slurm_keys=['JobName', 'Elapsed', 'ExitCode']): - """Return info dict from slurm job as given by sacct + @staticmethod + def get_job_info(job_id, slurm_keys=['JobName', 'Elapsed', 'ExitCode'], parse=True): + """Retrieve slurm job info as given by sacct + if parse is True, return the raw string from sacct else parse info into a dict. All possible keys are given by `sacct --helpformat`""" + # Get info from sacct - cmd = [ - "sacct", f"--format={', '.join(slurm_keys)}", "--parsable", "-j", - str(jobid) - ] - r = subprocess.run(cmd, capture_output=True) - - # Parse in a dictionnary before returning - # The inner most process should be the relevant one, hence the 1 index - slurm_info = r.stdout.split()[1].split(b'|') - return ({k: v.decode() for k, v in zip(slurm_keys, slurm_info)}) - + cmd = ["sacct", f"--format={', '.join(slurm_keys)}", "-j", str(job_id)] + + if parse: + cmd.append("--parsable") + + info_str = subprocess.run(cmd, capture_output=True, check=True).stdout + + if parse: + # Parse in a dictionnary before returning + # The inner most process should be the relevant one, hence the 1 index + slurm_info = info_str.split(b'\n')[1].split(b'|') + return {k: v.decode() for k, v in zip(slurm_keys, slurm_info)} + else: + return info_str.decode() + + def get_slurm_summary(self): + """get slurm info summary or all jobs of current chunk""" + + # Get job info for all jobs + self.slurm_info = {} + for job_name in self.jobs: + for job_id in self.job_ids['current'][job_name]: + self.slurm_info[job_name] = [] + self.slurm_info[job_name].append( + self.get_job_info(job_id, slurm_keys=info_requests.keys(), + parse=True) + ) + + def print_slurm_summary(self): + # Build table header and line format + # (could be done class-wide, not for each call) + headers = [] + hlines = [] + formats = [] + for k, l in self.info_requests.items(): + formats.append(f"{{{k}:>{l}.{l}}}") + headers.append(f"{k:>{l}.{l}}") + hlines.append("-"*l) + + table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) + line_format = " ".join(formats) + + print(" └── Slurm info of submitted jobs\n") + + for job_name in self.jobs: + print(f" └── {job_name}") + print(table_header) + for info in self.slurm_info[job_name]: + print(line_format.format(**info)) + + def check_chunk_success(self): + status = 0 + for job_name, info_list in self.slurm_info.items(): + for info in info_list: + if info['State'] != 'COMPLETED': + status += 1 + + if status > 0: + raise RuntimeError("One or more job failed") + class InvalidWorkflowType(Exception): pass diff --git a/run_chain.py b/run_chain.py index 116735b5..a4b275bf 100755 --- a/run_chain.py +++ b/run_chain.py @@ -10,6 +10,7 @@ import time import shutil import argparse +import inspect import jobs from jobs import tools @@ -234,7 +235,7 @@ def run_chunk(cfg, force, resume): skip = True else: print( - f' └── Process "{job_name}" for chunk "{cfg.chunk_id}"') + f' └── Submit "{job_name}" for chunk "{cfg.chunk_id}"') # Logfile settings cfg.logfile = cfg.log_working_dir / job_name @@ -251,16 +252,12 @@ def run_chunk(cfg, force, resume): # wait for previous chunk to be done cfg.wait_for_previous() - # - [ ] Matthieu : Monitor current chunk - # For each job check and print: - # - job id - # - dependencies - # - status (State) - # - elapsed time (Elapsed) - # - start time (Start) - # - end time (End) - # - partition (Partition) - # - number of nodes (NNodes) + # Current chunk Slurm summary + cfg.get_slurm_summary() + cfg.print_slurm_summary() + + # Check for success + cfg.check_chunk_success() # cycle cfg.job_ids['previous'] = cfg.job_ids['current'] From 622ecf00e7b1e938e7e1cc2d24d731d876379ebf Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 11 Jan 2024 15:06:12 +0000 Subject: [PATCH 138/245] GitHub Action: Apply Pep8-formatting --- config.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index ff13a850..a06644ef 100644 --- a/config.py +++ b/config.py @@ -10,8 +10,16 @@ class Config(): # Requested slurm info keys and corresponding printed width - info_requests = {'JobName': 10, 'JobID': 8, 'Partition': 9, 'NNodes': 6, - 'State': 14, 'Start': 13, 'End': 13, 'Elapsed': 9} + info_requests = { + 'JobName': 10, + 'JobID': 8, + 'Partition': 9, + 'NNodes': 6, + 'State': 14, + 'Start': 13, + 'End': 13, + 'Elapsed': 9 + } def __init__(self, casename): """Initialize an instance of the Config class. @@ -541,18 +549,20 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) @staticmethod - def get_job_info(job_id, slurm_keys=['JobName', 'Elapsed', 'ExitCode'], parse=True): + def get_job_info(job_id, + slurm_keys=['JobName', 'Elapsed', 'ExitCode'], + parse=True): """Retrieve slurm job info as given by sacct if parse is True, return the raw string from sacct else parse info into a dict. All possible keys are given by `sacct --helpformat`""" - + # Get info from sacct cmd = ["sacct", f"--format={', '.join(slurm_keys)}", "-j", str(job_id)] if parse: cmd.append("--parsable") - + info_str = subprocess.run(cmd, capture_output=True, check=True).stdout if parse: @@ -572,9 +582,9 @@ def get_slurm_summary(self): for job_id in self.job_ids['current'][job_name]: self.slurm_info[job_name] = [] self.slurm_info[job_name].append( - self.get_job_info(job_id, slurm_keys=info_requests.keys(), - parse=True) - ) + self.get_job_info(job_id, + slurm_keys=info_requests.keys(), + parse=True)) def print_slurm_summary(self): # Build table header and line format @@ -585,13 +595,13 @@ def print_slurm_summary(self): for k, l in self.info_requests.items(): formats.append(f"{{{k}:>{l}.{l}}}") headers.append(f"{k:>{l}.{l}}") - hlines.append("-"*l) - + hlines.append("-" * l) + table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) line_format = " ".join(formats) print(" └── Slurm info of submitted jobs\n") - + for job_name in self.jobs: print(f" └── {job_name}") print(table_header) @@ -607,7 +617,7 @@ def check_chunk_success(self): if status > 0: raise RuntimeError("One or more job failed") - + class InvalidWorkflowType(Exception): pass From 8d79b66d31e142ae44b62a107421b330136e5963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 11 Jan 2024 21:13:13 +0100 Subject: [PATCH 139/245] Add BASIC_PYHTON_JOB to missing jobs --- jobs/cosmo.py | 2 ++ jobs/icontools.py | 2 ++ jobs/prepare_icon.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 1bc70132..c48e4a9e 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -9,6 +9,8 @@ from . import tools from datetime import datetime, timedelta +BASIC_PYTHON_JOB = True + def set_cfg_variables(cfg): cfg.cosmo_base = cfg.chain_root / 'cosmo' diff --git a/jobs/icontools.py b/jobs/icontools.py index 2e86fb66..ad5006cc 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -7,6 +7,8 @@ import numpy as np from . import tools, prepare_icon +BASIC_PYTHON_JOB = True + def main(cfg): """ diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 2135db7e..d588536e 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -5,6 +5,8 @@ import logging from . import tools +BASIC_PYTHON_JOB = True + def set_cfg_variables(cfg): cfg.icon_base = cfg.chain_root / 'icon' From 6389bdc1034609de6d2bdbde332ca6592d0df2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 11 Jan 2024 21:16:11 +0100 Subject: [PATCH 140/245] Fix function arguments --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index a4b275bf..ef893b24 100755 --- a/run_chain.py +++ b/run_chain.py @@ -244,7 +244,7 @@ def run_chunk(cfg, force, resume): # Submit the job job = getattr(jobs, job_name) if hasattr(job, 'BASIC_PYTHON_JOB') and job.BASIC_PYTHON_JOB: - script = cfg.create_sbatch_script(job_name, logfile) + script = cfg.create_sbatch_script(job_name) cfg.submit(job_name, script) else: job.main(cfg) From 29fcbc0fce0720f99c11fa509e098c4f231d61cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 11 Jan 2024 21:34:27 +0100 Subject: [PATCH 141/245] Small fix and hint to KeyError --- config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index a06644ef..3d842d5c 100644 --- a/config.py +++ b/config.py @@ -583,7 +583,7 @@ def get_slurm_summary(self): self.slurm_info[job_name] = [] self.slurm_info[job_name].append( self.get_job_info(job_id, - slurm_keys=info_requests.keys(), + slurm_keys=self.info_requests.keys(), parse=True)) def print_slurm_summary(self): @@ -600,13 +600,13 @@ def print_slurm_summary(self): table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) line_format = " ".join(formats) - print(" └── Slurm info of submitted jobs\n") + print(" └── Slurm info of submitted jobs") for job_name in self.jobs: print(f" └── {job_name}") print(table_header) for info in self.slurm_info[job_name]: - print(line_format.format(**info)) + print(line_format.format(**info)) # KeyError: 'JobID' def check_chunk_success(self): status = 0 From 7de963d2f9f950bcab53c267ab48ce4b926af713 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 11 Jan 2024 20:34:52 +0000 Subject: [PATCH 142/245] GitHub Action: Apply Pep8-formatting --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 3d842d5c..e02c0b9e 100644 --- a/config.py +++ b/config.py @@ -606,7 +606,7 @@ def print_slurm_summary(self): print(f" └── {job_name}") print(table_header) for info in self.slurm_info[job_name]: - print(line_format.format(**info)) # KeyError: 'JobID' + print(line_format.format(**info)) # KeyError: 'JobID' def check_chunk_success(self): status = 0 From 87edb3b8f42b6344f83180f6c1e217aadb5f671d Mon Sep 17 00:00:00 2001 From: Matthieu Date: Fri, 12 Jan 2024 12:04:55 +0100 Subject: [PATCH 143/245] ref(slurm summary): streamline code a bit --- config.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index ff13a850..bb75c35a 100644 --- a/config.py +++ b/config.py @@ -563,26 +563,34 @@ def get_job_info(job_id, slurm_keys=['JobName', 'Elapsed', 'ExitCode'], parse=Tr else: return info_str.decode() - def get_slurm_summary(self): + def get_slurm_summary(self, info_keys=['JobName', 'JobID', 'Partition', 'NNodes', + 'State', 'Start', 'End', 'Elapsed']): """get slurm info summary or all jobs of current chunk""" + # Store requested keys in object + self.info_keys = info_keys + # Get job info for all jobs self.slurm_info = {} for job_name in self.jobs: for job_id in self.job_ids['current'][job_name]: self.slurm_info[job_name] = [] self.slurm_info[job_name].append( - self.get_job_info(job_id, slurm_keys=info_requests.keys(), - parse=True) + self.get_job_info(job_id, slurm_keys=info_keys, parse=True) ) def print_slurm_summary(self): + + # Width of printed slurm piece of information + info_width = {'JobName': 10, 'JobID': 8, 'Partition': 9, 'NNodes': 6, + 'State': 14, 'Start': 13, 'End': 13, 'Elapsed': 9} + # Build table header and line format - # (could be done class-wide, not for each call) headers = [] hlines = [] formats = [] - for k, l in self.info_requests.items(): + for k in self.info_keys: + l = info_width[k] formats.append(f"{{{k}:>{l}.{l}}}") headers.append(f"{k:>{l}.{l}}") hlines.append("-"*l) From 390b4995c1ef00ad22af523f3ce8a0e1316b501a Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 12 Jan 2024 11:07:02 +0000 Subject: [PATCH 144/245] GitHub Action: Apply Pep8-formatting --- config.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index 8e26d87c..8b59200b 100644 --- a/config.py +++ b/config.py @@ -573,28 +573,39 @@ def get_job_info(job_id, else: return info_str.decode() - def get_slurm_summary(self, info_keys=['JobName', 'JobID', 'Partition', 'NNodes', - 'State', 'Start', 'End', 'Elapsed']): + def get_slurm_summary(self, + info_keys=[ + 'JobName', 'JobID', 'Partition', 'NNodes', + 'State', 'Start', 'End', 'Elapsed' + ]): """get slurm info summary or all jobs of current chunk""" # Store requested keys in object self.info_keys = info_keys - + # Get job info for all jobs self.slurm_info = {} for job_name in self.jobs: for job_id in self.job_ids['current'][job_name]: self.slurm_info[job_name] = [] self.slurm_info[job_name].append( - self.get_job_info(job_id, slurm_keys=info_keys, parse=True) - ) + self.get_job_info(job_id, slurm_keys=info_keys, + parse=True)) def print_slurm_summary(self): # Width of printed slurm piece of information - info_width = {'JobName': 10, 'JobID': 8, 'Partition': 9, 'NNodes': 6, - 'State': 14, 'Start': 13, 'End': 13, 'Elapsed': 9} - + info_width = { + 'JobName': 10, + 'JobID': 8, + 'Partition': 9, + 'NNodes': 6, + 'State': 14, + 'Start': 13, + 'End': 13, + 'Elapsed': 9 + } + # Build table header and line format headers = [] hlines = [] From dffb13e2ce68ba7cc391181a67cacdc05c36e585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 12 Jan 2024 14:59:16 +0100 Subject: [PATCH 145/245] Some settings for cosmo-ghg case --- cases/cosmo-ghg-test/config.yaml | 2 ++ jobs/int2lm.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cases/cosmo-ghg-test/config.yaml b/cases/cosmo-ghg-test/config.yaml index 163e8dd8..b954e0f2 100644 --- a/cases/cosmo-ghg-test/config.yaml +++ b/cases/cosmo-ghg-test/config.yaml @@ -2,6 +2,8 @@ workflow: cosmo-ghg constraint: gpu +run_on: gpu +compute_queue: normal ntasks_per_node: 12 restart_step: PT6H startdate: 2015-01-01T00:00:00Z diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 97578178..5fe59682 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -5,6 +5,7 @@ import logging import shutil import subprocess +import pytz from datetime import datetime from . import tools, prepare_cosmo @@ -111,7 +112,7 @@ def main(cfg): input_art_filename) # Change of soil model from TERRA to TERRA multi-layer on 2 Aug 2007 - if cfg.startdate_sim < datetime(2007, 8, 2): + if cfg.startdate_sim < datetime(2007, 8, 2, tzinfo=pytz.UTC): multi_layer = ".FALSE." else: multi_layer = ".TRUE." From 9da09f4a174d37fe0793268419b848e07cdacc49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 12 Jan 2024 14:59:48 +0100 Subject: [PATCH 146/245] afterany -> afterok for wait job --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index e02c0b9e..2c0d7919 100644 --- a/config.py +++ b/config.py @@ -540,7 +540,7 @@ def wait_for_previous(self): f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', - f'#SBATCH --dependency=afterany:{dep_str}', '', '# Do nothing', + f'#SBATCH --dependency=afterok:{dep_str}', '', '# Do nothing', 'exit 0' ] with open(job_file, mode='w') as wait_job: From 4f40ec0ba16dec8652526b72bc5bed5b4e42007e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 12 Jan 2024 15:34:14 +0100 Subject: [PATCH 147/245] Comment some function calls --- run_chain.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/run_chain.py b/run_chain.py index ef893b24..f8fac92d 100755 --- a/run_chain.py +++ b/run_chain.py @@ -225,7 +225,7 @@ def run_chunk(cfg, force, resume): if cfg.is_async: # Empty curent job ids - cfg.job_ids['current'] = {} + # cfg.job_ids['current'] = {} # Submit current chunk for job_name in cfg.jobs: @@ -249,17 +249,17 @@ def run_chunk(cfg, force, resume): else: job.main(cfg) - # wait for previous chunk to be done + # Wait for previous chunk to be done cfg.wait_for_previous() # Current chunk Slurm summary cfg.get_slurm_summary() - cfg.print_slurm_summary() + # cfg.print_slurm_summary() # Check for success - cfg.check_chunk_success() + # cfg.check_chunk_success() - # cycle + # Cycle cfg.job_ids['previous'] = cfg.job_ids['current'] else: # run jobs (if required) From c1b1ca9f1c715e3ba427636ff4a1b0ccaac8c570 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:04:03 +0100 Subject: [PATCH 148/245] add:ref: remove unused `info_requests` + print failing jobs --- config.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/config.py b/config.py index 8e26d87c..fe31b995 100644 --- a/config.py +++ b/config.py @@ -9,18 +9,6 @@ class Config(): - # Requested slurm info keys and corresponding printed width - info_requests = { - 'JobName': 10, - 'JobID': 8, - 'Partition': 9, - 'NNodes': 6, - 'State': 14, - 'Start': 13, - 'End': 13, - 'Elapsed': 9 - } - def __init__(self, casename): """Initialize an instance of the Config class. @@ -618,13 +606,15 @@ def print_slurm_summary(self): def check_chunk_success(self): status = 0 + failed_jobs = [] for job_name, info_list in self.slurm_info.items(): for info in info_list: if info['State'] != 'COMPLETED': + failed_jobs.append(job_name) status += 1 if status > 0: - raise RuntimeError("One or more job failed") + raise RuntimeError(f"The following job(s) failed: {failed_jobs}") class InvalidWorkflowType(Exception): From f001103bde67c7b378418ac09f575f8e913d6bfe Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:10:12 +0100 Subject: [PATCH 149/245] fix: job summary for previous chunk, not current --- config.py | 8 ++++---- run_chain.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index b203324e..fe16aac2 100644 --- a/config.py +++ b/config.py @@ -528,7 +528,7 @@ def wait_for_previous(self): f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', - f'#SBATCH --dependency=afterok:{dep_str}', '', '# Do nothing', + f'#SBATCH --dependency=afterany:{dep_str}', '', '# Do nothing', 'exit 0' ] with open(job_file, mode='w') as wait_job: @@ -566,7 +566,7 @@ def get_slurm_summary(self, 'JobName', 'JobID', 'Partition', 'NNodes', 'State', 'Start', 'End', 'Elapsed' ]): - """get slurm info summary or all jobs of current chunk""" + """get slurm info summary or all jobs of previous chunk""" # Store requested keys in object self.info_keys = info_keys @@ -574,7 +574,7 @@ def get_slurm_summary(self, # Get job info for all jobs self.slurm_info = {} for job_name in self.jobs: - for job_id in self.job_ids['current'][job_name]: + for job_id in self.job_ids['previous'][job_name]: self.slurm_info[job_name] = [] self.slurm_info[job_name].append( self.get_job_info(job_id, slurm_keys=info_keys, @@ -607,7 +607,7 @@ def print_slurm_summary(self): table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) line_format = " ".join(formats) - print(" └── Slurm info of submitted jobs") + print(" └── Slurm info of previous submitted jobs") for job_name in self.jobs: print(f" └── {job_name}") diff --git a/run_chain.py b/run_chain.py index f8fac92d..4b29780b 100755 --- a/run_chain.py +++ b/run_chain.py @@ -254,10 +254,10 @@ def run_chunk(cfg, force, resume): # Current chunk Slurm summary cfg.get_slurm_summary() - # cfg.print_slurm_summary() + cfg.print_slurm_summary() # Check for success - # cfg.check_chunk_success() + cfg.check_chunk_success() # Cycle cfg.job_ids['previous'] = cfg.job_ids['current'] From 91e22511418def282e6ad67abf998c4d6d425381 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:12:48 +0100 Subject: [PATCH 150/245] fix: reactivate empty current job ids at beginning of chunk --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 4b29780b..e1eb8cd9 100755 --- a/run_chain.py +++ b/run_chain.py @@ -225,7 +225,7 @@ def run_chunk(cfg, force, resume): if cfg.is_async: # Empty curent job ids - # cfg.job_ids['current'] = {} + cfg.job_ids['current'] = {} # Submit current chunk for job_name in cfg.jobs: From 1cfcd20dcab5440210d41cb43c0ec2588b23ea2b Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:18:57 +0100 Subject: [PATCH 151/245] fix: only wait and monitor if not very first chunk --- run_chain.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/run_chain.py b/run_chain.py index e1eb8cd9..9384161a 100755 --- a/run_chain.py +++ b/run_chain.py @@ -249,15 +249,19 @@ def run_chunk(cfg, force, resume): else: job.main(cfg) - # Wait for previous chunk to be done - cfg.wait_for_previous() + - # Current chunk Slurm summary - cfg.get_slurm_summary() - cfg.print_slurm_summary() + # Return if very first chunk + if cfg.job_ids['previous']: + # Wait for previous chunk to be done + cfg.wait_for_previous() - # Check for success - cfg.check_chunk_success() + # Current chunk Slurm summary + cfg.get_slurm_summary() + cfg.print_slurm_summary() + + # Check for success + cfg.check_chunk_success() # Cycle cfg.job_ids['previous'] = cfg.job_ids['current'] From db2b39348072fe7f4bb2c8a4826045c2f76a5ed3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 15 Jan 2024 09:19:45 +0000 Subject: [PATCH 152/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index 9384161a..801b9249 100755 --- a/run_chain.py +++ b/run_chain.py @@ -249,8 +249,6 @@ def run_chunk(cfg, force, resume): else: job.main(cfg) - - # Return if very first chunk if cfg.job_ids['previous']: # Wait for previous chunk to be done From 3d0fb4856d65dffb492e3022059533b76a2faac2 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:50:33 +0100 Subject: [PATCH 153/245] ref: move icon-art error handling in slurm job itself --- cases/icon-art-global-test/icon_runjob.cfg | 10 +++++++++- config.py | 10 ++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index 1e324a59..09e09593 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -388,7 +388,15 @@ EOF # ---------------------------------------------------------------------- # run the model! # ---------------------------------------------------------------------- - srun ./icon.exe +handle_error(){ + # Check for invalid pointer error at the end of icon-art + if grep -q "free(): invalid pointer" {cfg.logfile} && grep -q "clean-up finished" {cfg.logfile}; then + exit 0 + else + exit 1 + fi +} + srun ./icon.exe || handle_error() diff --git a/config.py b/config.py index fe16aac2..5fe064f0 100644 --- a/config.py +++ b/config.py @@ -471,13 +471,7 @@ def submit(self, job_name, script, add_dep=None, logfile=None): return job_id def check_job(self, exitcode, logfile=None): - """Check the exitcode returned by a job. - In case of ICON-ART, ignore the "invalid pointer" error on a successful run. - """ - if logfile and tools.grep("ART: ", logfile)['success'] and \ - tools.grep("free(): invalid pointer", logfile)['success'] and \ - tools.grep("clean-up finished", logfile)['success']: - exitcode = 0 + """Check the exitcode returned by a job.""" if exitcode != 0: raise RuntimeError(f"sbatch returned exitcode {exitcode}") @@ -613,7 +607,7 @@ def print_slurm_summary(self): print(f" └── {job_name}") print(table_header) for info in self.slurm_info[job_name]: - print(line_format.format(**info)) # KeyError: 'JobID' + print(line_format.format(**info)) def check_chunk_success(self): status = 0 From 648eb8d5a5acd21de0b33d4f3df8a41309562074 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:52:16 +0100 Subject: [PATCH 154/245] fix: remove BASIC_PYTHON_JOB workaround for icon --- jobs/icon.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 4bc34661..3fd55ddf 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -4,10 +4,7 @@ import logging from . import tools, prepare_icon -BASIC_PYTHON_JOB = True -# FIXME: BASIC_PYTHON_JOB should be False, but with ICON-ART the sbatch job is -# always failing because of the "invalid pointer" error. This is caught -# if this module itself is submitted as an sbatch job. +BASIC_PYTHON_JOB = False def main(cfg): From 97b6cfff3ca9d68bf15fff72023def6d47b9b402 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:56:02 +0100 Subject: [PATCH 155/245] fix(icon.py): escape curly brackets for string formatting --- cases/icon-art-global-test/icon_runjob.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index 09e09593..c952eb30 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -388,14 +388,14 @@ EOF # ---------------------------------------------------------------------- # run the model! # ---------------------------------------------------------------------- -handle_error(){ +handle_error()\{ # Check for invalid pointer error at the end of icon-art if grep -q "free(): invalid pointer" {cfg.logfile} && grep -q "clean-up finished" {cfg.logfile}; then exit 0 else exit 1 fi -} +\} srun ./icon.exe || handle_error() From 6c3b54029bab0445f6d74e98fd3c841573fd3245 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 15 Jan 2024 10:57:32 +0100 Subject: [PATCH 156/245] fix(icon): escape curly brackets in python way --- cases/icon-art-global-test/icon_runjob.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index c952eb30..8059f12b 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -388,14 +388,14 @@ EOF # ---------------------------------------------------------------------- # run the model! # ---------------------------------------------------------------------- -handle_error()\{ +handle_error(){{ # Check for invalid pointer error at the end of icon-art if grep -q "free(): invalid pointer" {cfg.logfile} && grep -q "clean-up finished" {cfg.logfile}; then exit 0 else exit 1 fi -\} +}} srun ./icon.exe || handle_error() From ea59d356f2f7868f76be61f04d6e786228e65ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 15 Jan 2024 11:42:55 +0100 Subject: [PATCH 157/245] Add walltime and remove conda activation --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 5fe064f0..d37b9e9f 100644 --- a/config.py +++ b/config.py @@ -480,10 +480,12 @@ def create_sbatch_script(self, job_name): """Create an sbatch script to launch jobs individually. Use run_chain.py arguments to submit those jobs. """ + walltime = self.walltime[job_name] if self.walltime[job_name] else "00:30:00" script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}_{self.chunk_id}"', '#SBATCH --nodes=1', + f'#SBATCH --time={walltime}', f'#SBATCH --output={self.logfile}', '#SBATCH --open-mode=append', f'#SBATCH --account={self.compute_account}', @@ -491,8 +493,6 @@ def create_sbatch_script(self, job_name): f'#SBATCH --constraint={self.constraint}', '', f'cd {self.chain_src_dir}', - 'eval "$(conda shell.bash hook)"', - 'conda activate proc-chain', f'./run_chain.py {self.casename} -j {job_name} -c {self.chunk_id} -f -s --no-logging', '', ] From b14e29fa325a09ac5207d0f4eb9309befeafc4c5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 15 Jan 2024 10:43:20 +0000 Subject: [PATCH 158/245] GitHub Action: Apply Pep8-formatting --- config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index d37b9e9f..08361896 100644 --- a/config.py +++ b/config.py @@ -480,7 +480,8 @@ def create_sbatch_script(self, job_name): """Create an sbatch script to launch jobs individually. Use run_chain.py arguments to submit those jobs. """ - walltime = self.walltime[job_name] if self.walltime[job_name] else "00:30:00" + walltime = self.walltime[job_name] if self.walltime[ + job_name] else "00:30:00" script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}_{self.chunk_id}"', From 4a50fa06c37e708007b7628982e0dad9e5df24ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 15 Jan 2024 11:47:50 +0100 Subject: [PATCH 159/245] Set walltime correctly --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index d37b9e9f..a727be90 100644 --- a/config.py +++ b/config.py @@ -480,7 +480,7 @@ def create_sbatch_script(self, job_name): """Create an sbatch script to launch jobs individually. Use run_chain.py arguments to submit those jobs. """ - walltime = self.walltime[job_name] if self.walltime[job_name] else "00:30:00" + walltime = getattr(self, 'walltime', {}).get(job_name, "00:30:00") script_lines = [ '#!/usr/bin/env bash', f'#SBATCH --job-name="{job_name}_{self.chunk_id}"', From e9f4d710c87b8ffd1a888c82fce09eabce791d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 10:06:14 +0100 Subject: [PATCH 160/245] Fix call to handle_error function --- cases/icon-art-global-test/icon_runjob.cfg | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index 8059f12b..cd0508d9 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -396,22 +396,4 @@ handle_error(){{ exit 1 fi }} - srun ./icon.exe || handle_error() - - - -# ! output_nml: specifies an output stream -------------------------------------- -# &output_nml -# filetype = 4 ! output format: 2=GRIB2, 4=NETCDFv2 -# dom = -1 ! write all domains -# output_bounds = 0., 2678400., 3600. ! start, end, increment -# steps_per_file = 1 ! number of steps per file -# mode = 1 ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis) -# include_last = .TRUE. -# output_filename = 'ICON-ART' -# filename_format = '{cfg.icon_output}/_latlon_' ! file name base -# remap = 1 ! 1: remap to lat-lon grid -# reg_lon_def = -179.,2,179 -# reg_lat_def = 90.,-1,-90. -# ml_varlist = 'z_ifc','z_mc','pres','pres_sfc','qc','rh','rho','temp','u','v','w','group:ART_CHEMISTRY', -# / + srun ./icon.exe || handle_error From 16d5b2d70f900fc585417d3018609b43c4777a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 10:19:26 +0100 Subject: [PATCH 161/245] Set walltimes for icon workflows --- cases/icon-art-global-test/config.yaml | 6 +++++- cases/icon-art-global-test/icon_runjob.cfg | 4 ++-- cases/icon-art-oem-test/config.yaml | 7 +++++++ cases/icon-art-oem-test/icon_runjob.cfg | 12 ++++++++++-- cases/icon-seq-test/config.yaml | 3 +++ cases/icon-seq-test/icon_runjob.cfg | 2 +- cases/icon-test/config.yaml | 6 +++++- cases/icon-test/icon_runjob.cfg | 2 +- 8 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cases/icon-art-global-test/config.yaml b/cases/icon-art-global-test/config.yaml index 5b49bd38..2fe3f096 100644 --- a/cases/icon-art-global-test/config.yaml +++ b/cases/icon-art-global-test/config.yaml @@ -28,6 +28,11 @@ species_global_nudging: False species2nudge: [] nudging_step: 6 +walltime: + prepare_icon: '00:15:00' + prepare_art_global: '00:10:00' + icon: '00:05:00' + era5: inicond: False global_nudging: False @@ -67,7 +72,6 @@ icon: species_nudgingjob: icon_species_nudging.sh output_writing_step: 6 compute_queue: normal - walltime: '00:10:00' np_tot: 4 np_io: 1 np_restart: 1 diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index cd0508d9..2c7db9d3 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" #SBATCH --account={cfg.compute_account} -#SBATCH --time={cfg.icon_walltime} +#SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} #SBATCH --ntasks-per-node={cfg.ntasks_per_node} #SBATCH --partition={cfg.compute_queue} @@ -396,4 +396,4 @@ handle_error(){{ exit 1 fi }} - srun ./icon.exe || handle_error +srun ./icon.exe || handle_error diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index 06d678d4..475d326a 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -21,6 +21,13 @@ filename_format: _DOM_ lateral_boundary_grid_order: lateral_boundary art_input_folder: ./input/icon-art-oem/ART +walltime: + prepare_icon: '00:10:00' + icontools: '00:30:00' + prepare_art: '00:10:00' + prepare_art_oem: '00:10:00' + icon: '00:30:00' + meteo: dir: ./input/meteo prefix: ifs_ diff --git a/cases/icon-art-oem-test/icon_runjob.cfg b/cases/icon-art-oem-test/icon_runjob.cfg index 3e69720c..4b3190b0 100644 --- a/cases/icon-art-oem-test/icon_runjob.cfg +++ b/cases/icon-art-oem-test/icon_runjob.cfg @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" #SBATCH --account={cfg.compute_account} -#SBATCH --time={cfg.icon_walltime} +#SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} #SBATCH --ntasks-per-node={cfg.ntasks_per_node} #SBATCH --partition={cfg.compute_queue} @@ -368,4 +368,12 @@ EOF # ---------------------------------------------------------------------- # run the model! # ---------------------------------------------------------------------- - srun ./icon.exe +handle_error(){{ + # Check for invalid pointer error at the end of icon-art + if grep -q "free(): invalid pointer" {cfg.logfile} && grep -q "clean-up finished" {cfg.logfile}; then + exit 0 + else + exit 1 + fi +}} +srun ./icon.exe || handle_error diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml index f1b520d1..a496e575 100644 --- a/cases/icon-seq-test/config.yaml +++ b/cases/icon-seq-test/config.yaml @@ -24,6 +24,9 @@ output_filename: NWP_LAM filename_format: _DOM_ lateral_boundary_grid_order: lateral_boundary +walltime: + icon: '00:30:00' + meteo: dir: ./input/meteo prefix: ifs_ diff --git a/cases/icon-seq-test/icon_runjob.cfg b/cases/icon-seq-test/icon_runjob.cfg index 09ff2c3a..f0ba8453 100755 --- a/cases/icon-seq-test/icon_runjob.cfg +++ b/cases/icon-seq-test/icon_runjob.cfg @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}" #SBATCH --account={cfg.compute_account} -#SBATCH --time={cfg.icon_walltime} +#SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} #SBATCH --ntasks-per-node={cfg.ntasks_per_node} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-test/config.yaml b/cases/icon-test/config.yaml index 261a0871..06065e42 100644 --- a/cases/icon-test/config.yaml +++ b/cases/icon-test/config.yaml @@ -18,6 +18,11 @@ output_filename: NWP_LAM filename_format: _DOM_ lateral_boundary_grid_order: lateral_boundary +walltime: + prepare_icon: '00:10:00' + icontools: '00:30:00' + icon: '00:30:00' + meteo: dir: ./input/meteo prefix: ifs_ @@ -44,7 +49,6 @@ icon: binary_file: ./ext/icon/bin/icon runjob_filename: icon_runjob.cfg compute_queue: normal - walltime: '00:10:00' np_tot: 8 np_io: 1 np_restart: 1 diff --git a/cases/icon-test/icon_runjob.cfg b/cases/icon-test/icon_runjob.cfg index 09ff2c3a..f0ba8453 100755 --- a/cases/icon-test/icon_runjob.cfg +++ b/cases/icon-test/icon_runjob.cfg @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}" #SBATCH --account={cfg.compute_account} -#SBATCH --time={cfg.icon_walltime} +#SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} #SBATCH --ntasks-per-node={cfg.ntasks_per_node} #SBATCH --partition={cfg.compute_queue} From 2b99c66dfc354716bfb099e6a8be60f27f4a6f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 10:31:25 +0100 Subject: [PATCH 162/245] Debug --- run_chain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run_chain.py b/run_chain.py index 801b9249..ca964318 100755 --- a/run_chain.py +++ b/run_chain.py @@ -226,6 +226,7 @@ def run_chunk(cfg, force, resume): if cfg.is_async: # Empty curent job ids cfg.job_ids['current'] = {} + print(cfg.job_ids) # Submit current chunk for job_name in cfg.jobs: @@ -248,6 +249,7 @@ def run_chunk(cfg, force, resume): cfg.submit(job_name, script) else: job.main(cfg) + print(cfg.job_ids) # Return if very first chunk if cfg.job_ids['previous']: @@ -256,6 +258,8 @@ def run_chunk(cfg, force, resume): # Current chunk Slurm summary cfg.get_slurm_summary() + print(cfg.job_ids) + print(cfg.slurm_info) cfg.print_slurm_summary() # Check for success From eb3d856fbdfa31d1fec8d1396390165559745f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 10:41:53 +0100 Subject: [PATCH 163/245] Remove job_ids override --- jobs/prepare_icon.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index d588536e..7c2f1ca3 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -46,8 +46,6 @@ def set_cfg_variables(cfg): cfg.startdate_sim_yyyymmdd_hh = cfg.startdate_sim.strftime('%Y%m%d_%H') - cfg.job_ids['current']['prepare_icon'] = [] - def main(cfg): """ From c5cce6a648b8d8cbe77be92ba31565e1f31f838c Mon Sep 17 00:00:00 2001 From: Matthieu Date: Tue, 16 Jan 2024 10:43:34 +0100 Subject: [PATCH 164/245] ref(config): clean up --- config.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 5fe064f0..f1cbba9a 100644 --- a/config.py +++ b/config.py @@ -51,11 +51,11 @@ def __init__(self, casename): self.set_account() self.chain_src_dir = Path.cwd() - self.case_path = self.chain_src_dir / 'cases' / casename + self.case_path = self.chain_src_dir / 'cases' / self.casename self.work_root = self.chain_src_dir / 'work' # User-defined attributes from config file - self.load_config_file(casename) + self.load_config_file() # Set case root self.case_root = self.work_root / self.casename @@ -66,17 +66,12 @@ def __init__(self, casename): # Specific settings based on the node type ('gpu' or 'mc') self.set_node_info() - def load_config_file(self, casename): + def load_config_file(self): """Load configuration settings from a YAML file and set them as attributes. This method reads the configuration settings from a YAML file located in the 'cases/casename' directory and sets them as attributes of the instance. - Parameters - ---------- - casename : str - Name of the folder in 'cases/' where the configuration files are stored. - Returns ------- Config @@ -94,17 +89,17 @@ def load_config_file(self, casename): existing case directories. The method directly assigns values from the configuration file to instance attributes for easy access. """ - cfg_file = Path('cases', casename, 'config.yaml').resolve() + cfg_file = Path('cases', self.casename, 'config.yaml').resolve() if not cfg_file.is_file(): all_cases = [ path.name for path in os.scandir('cases') if path.is_dir() ] - closest_name = min([(tools.levenshtein(casename, name), name) + closest_name = min([(tools.levenshtein(self.casename, name), name) for name in all_cases], key=lambda x: x[0])[1] raise FileNotFoundError( - f"Case-directory '{casename}' not found, did you mean '{closest_name}'?" + f"Case-directory '{self.casename}' not found, did you mean '{closest_name}'?" ) try: @@ -118,11 +113,6 @@ def load_config_file(self, casename): for key, value in cfg_data.items(): setattr(self, key, value) - # rename the workflow attribute to avoid name clash - self.workflow_name = self.workflow - - return self - def set_account(self): """Set the compute account based on user information. @@ -193,6 +183,7 @@ def set_workflow(self): """set workflow and async attr, initiate job ids dict""" # If a workflow name is specified, load from workflows.yaml if isinstance(self.workflow, str): + self.workflow_name = self.workflow with open('workflows.yaml') as file: workflows = yaml.safe_load(file) self.workflow = workflows[self.workflow_name] From dee4c0e71c83ef855ff8cfd1a70d4bdfdab866cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 10:45:58 +0100 Subject: [PATCH 165/245] Add smaller walltime to wait job --- config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index a727be90..ac7f497b 100644 --- a/config.py +++ b/config.py @@ -518,7 +518,8 @@ def wait_for_previous(self): dep_str = ':'.join(map(str, dep_ids)) script_lines = [ '#!/usr/bin/env bash', '#SBATCH --job-name="wait"', - '#SBATCH --nodes=1', f'#SBATCH --output={log_file}', + '#SBATCH --nodes=1', '#SBATCH --time=00:01:00', + f'#SBATCH --output={log_file}', f'#SBATCH --account={self.compute_account}', f'#SBATCH --partition={self.compute_queue}', f'#SBATCH --constraint={self.constraint}', From add761eb0c2ea6171af556031323b1415cbd45ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 10:47:04 +0100 Subject: [PATCH 166/245] Remove prints --- run_chain.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/run_chain.py b/run_chain.py index ca964318..801b9249 100755 --- a/run_chain.py +++ b/run_chain.py @@ -226,7 +226,6 @@ def run_chunk(cfg, force, resume): if cfg.is_async: # Empty curent job ids cfg.job_ids['current'] = {} - print(cfg.job_ids) # Submit current chunk for job_name in cfg.jobs: @@ -249,7 +248,6 @@ def run_chunk(cfg, force, resume): cfg.submit(job_name, script) else: job.main(cfg) - print(cfg.job_ids) # Return if very first chunk if cfg.job_ids['previous']: @@ -258,8 +256,6 @@ def run_chunk(cfg, force, resume): # Current chunk Slurm summary cfg.get_slurm_summary() - print(cfg.job_ids) - print(cfg.slurm_info) cfg.print_slurm_summary() # Check for success From 780c66dbd2564ef0223af47fbe00df7fb250e3cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 15:43:28 +0100 Subject: [PATCH 167/245] Some fixes for cosmo case --- jobs/cosmo.py | 44 ++----------------------------------------- jobs/oem.py | 4 ++-- jobs/online_vprm.py | 3 ++- jobs/prepare_cosmo.py | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 45 deletions(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index c48e4a9e..87749f8c 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -4,51 +4,13 @@ import logging import os import subprocess -import csv from .tools import write_cosmo_input_ghg -from . import tools +from . import tools, prepare_cosmo from datetime import datetime, timedelta BASIC_PYTHON_JOB = True -def set_cfg_variables(cfg): - cfg.cosmo_base = cfg.chain_root / 'cosmo' - cfg.cosmo_input = cfg.chain_root / 'cosmo' / 'input' - cfg.cosmo_run = cfg.chain_root / 'cosmo' / 'run' - cfg.cosmo_output = cfg.chain_root / 'cosmo' / 'output' - cfg.cosmo_output_reduced = cfg.chain_root / 'cosmo' / 'output_reduced' - - # Number of tracers - if 'tracers' in cfg.workflow['features']: - tracer_csvfile = cfg.chain_src_dir / 'cases' / cfg.casename / 'cosmo_tracers.csv' - if tracer_csvfile.is_file(): - with open(tracer_csvfile, 'r') as csv_file: - reader = csv.DictReader(csv_file, delimiter=',') - reader = [r for r in reader if r[''] != '#'] - cfg.in_tracers = len(reader) - else: - raise FileNotFoundError(f"File not found: {tracer_csvfile}") - - # tracer_start namelist parameter for spinup simulation - if hasattr(cfg, 'spinup'): - if cfg.first_one: - cfg.tracer_start = 0 - else: - cfg.tracer_start = cfg.spinup - else: - cfg.tracer_start = 0 - - # asynchronous I/O - if hasattr(cfg, 'cfg.cosmo_np_io'): - if cfg.cosmo_np_io == 0: - cfg.lasync_io = '.FALSE.' - cfg.num_iope_percomm = 0 - else: - cfg.lasync_io = '.TRUE.' - cfg.num_iope_percomm = 1 - - def main(cfg): """Setup the namelists for a COSMO tracer run and submit the job to the queue. @@ -78,7 +40,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - set_cfg_variables(cfg) + prepare_cosmo.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("cosmo") @@ -101,8 +63,6 @@ def main(cfg): tools.create_dir(ini_dir, "cosmo_input_initial") startfiletime = datetime.strptime(cfg.laf_startfile[-10:], "%Y%m%d%H") if cfg.startdate_sim >= startfiletime: - starttime_last = cfg.startdate_sim - timedelta( - hours=cfg.restart_step) work_root = os.path.dirname(os.path.dirname(cfg.chain_root)) last_output_path = os.path.join(work_root, cfg.casename, cfg.chunk_id_prev, 'cosmo', diff --git a/jobs/oem.py b/jobs/oem.py index 9ff1eb62..64d93dc8 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -4,7 +4,7 @@ import os import logging -from . import tools, cosmo +from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True @@ -24,7 +24,7 @@ def main(cfg): If an error occurs during the process. """ tools.change_logfile(cfg.logfile) - cosmo.set_cfg_variables(cfg) + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("oem") oem_dir = cfg.oem['dir'] diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index a6488549..7808ad69 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -4,7 +4,7 @@ import os import logging -from . import tools +from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True @@ -18,6 +18,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ + prepare_cosmo.set_cfg_variables() tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("online_vprm") dest_modis = 'modis.nc' diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index f339a366..155d9d4d 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -3,6 +3,7 @@ from pathlib import Path import logging +import csv from datetime import timedelta from . import tools @@ -12,6 +13,40 @@ def set_cfg_variables(cfg): cfg.int2lm_root = cfg.chain_root / 'int2lm' cfg.int2lm_input = cfg.int2lm_root / 'input' + cfg.cosmo_base = cfg.chain_root / 'cosmo' + cfg.cosmo_input = cfg.chain_root / 'cosmo' / 'input' + cfg.cosmo_run = cfg.chain_root / 'cosmo' / 'run' + cfg.cosmo_output = cfg.chain_root / 'cosmo' / 'output' + cfg.cosmo_output_reduced = cfg.chain_root / 'cosmo' / 'output_reduced' + + # Number of tracers + if 'tracers' in cfg.workflow['features']: + tracer_csvfile = cfg.chain_src_dir / 'cases' / cfg.casename / 'cosmo_tracers.csv' + if tracer_csvfile.is_file(): + with open(tracer_csvfile, 'r') as csv_file: + reader = csv.DictReader(csv_file, delimiter=',') + reader = [r for r in reader if r[''] != '#'] + cfg.in_tracers = len(reader) + else: + raise FileNotFoundError(f"File not found: {tracer_csvfile}") + + # tracer_start namelist parameter for spinup simulation + if hasattr(cfg, 'spinup'): + if cfg.first_one: + cfg.tracer_start = 0 + else: + cfg.tracer_start = cfg.spinup + else: + cfg.tracer_start = 0 + + # asynchronous I/O + if hasattr(cfg, 'cfg.cosmo_np_io'): + if cfg.cosmo_np_io == 0: + cfg.lasync_io = '.FALSE.' + cfg.num_iope_percomm = 0 + else: + cfg.lasync_io = '.TRUE.' + cfg.num_iope_percomm = 1 def main(cfg): From 0e6de773141dc2a7002cb88e113ec77818ef499d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 15:50:37 +0100 Subject: [PATCH 168/245] Remove model check --- jobs/biofluxes.py | 1 - jobs/photo_rate.py | 1 - 2 files changed, 2 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 4bbfbc60..2fde767b 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -19,7 +19,6 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - tools.check_model(cfg, 'cosmo-ghg') prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("biofluxes") diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index 218dcaf7..0814b753 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -23,7 +23,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.change_logfile(cfg.logfile) - tools.check_model(cfg, 'cosmo-art') launch_time = cfg.init_time_logging("photo_rate") logging.info("Copying photolysis-rate file from {} to {}".format( From 2e6a45fc41eb336411436f78ced5d5e9ab9138c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 15:52:51 +0100 Subject: [PATCH 169/245] Don't pass logfile anymore --- jobs/icon.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 3fd55ddf..963797c8 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -61,14 +61,12 @@ def main(cfg): # Write run script (run_icon.job) template = (cfg.case_path / cfg.icon_runjob_filename).read_text() script_str = template.format(cfg=cfg, - inidata_filename=inidata_filename, - logfile=cfg.logfile, - logfile_finish=cfg.logfile_finish) + inidata_filename=inidata_filename) script = (cfg.icon_work / 'run_icon.job') script.write_text(script_str) # Submit run script - cfg.submit('icon', script, logfile=cfg.logfile) + cfg.submit('icon', script) if BASIC_PYTHON_JOB: cfg.finish_time_logging("icon", launch_time) From 6f88cdfe56d90e7d3519dae40f8bc728389b3f77 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 16 Jan 2024 14:53:17 +0000 Subject: [PATCH 170/245] GitHub Action: Apply Pep8-formatting --- jobs/icon.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index 963797c8..0461f3fe 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -60,8 +60,7 @@ def main(cfg): # Write run script (run_icon.job) template = (cfg.case_path / cfg.icon_runjob_filename).read_text() - script_str = template.format(cfg=cfg, - inidata_filename=inidata_filename) + script_str = template.format(cfg=cfg, inidata_filename=inidata_filename) script = (cfg.icon_work / 'run_icon.job') script.write_text(script_str) From 5b560a025f375a51a1c937636a51e769712f7f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 15:57:52 +0100 Subject: [PATCH 171/245] Submit int2lm and cosmo jobs correctly --- jobs/cosmo.py | 12 ++++-------- jobs/int2lm.py | 11 +++-------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 87749f8c..7018ebba 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -188,8 +188,8 @@ def main(cfg): with open(runscript_file) as input_file: cosmo_runscript = input_file.read() - output_file = os.path.join(cfg.cosmo_run, "run.job") - with open(output_file, "w") as outf: + script = (cfg.int2lm_work / 'run_cosmo.job') + with open(script, "w") as outf: outf.write( cosmo_runscript.format(cfg=cfg, **cfg.cosmo, @@ -197,11 +197,7 @@ def main(cfg): logfile=cfg.logfile, logfile_finish=cfg.logfile_finish)) - result = subprocess.run( - ["sbatch", "--wait", - os.path.join(cfg.cosmo_run, 'run.job')]) - exitcode = result.returncode - if exitcode != 0: - raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + # Submit job + cfg.submit('cosmo', script) cfg.finish_time_logging("cosmo", launch_time) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 5fe59682..f185e4e5 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -143,8 +143,8 @@ def main(cfg): cfg.int2lm['runjob_filename'])) as input_file: int2lm_runscript = input_file.read() - output_file = os.path.join(int2lm_run, "run.job") - with open(output_file, "w") as outf: + script = (cfg.int2lm_work / 'run_int2lm.job') + with open(script, "w") as outf: outf.write( int2lm_runscript.format(cfg=cfg, **cfg.int2lm, @@ -157,11 +157,6 @@ def main(cfg): logfile_finish=cfg.logfile_finish)) # Submit job - result = subprocess.run( - ["sbatch", "--wait", - os.path.join(int2lm_run, "run.job")]) - exitcode = result.returncode - if exitcode != 0: - raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + cfg.submit('int2lm', script) cfg.finish_time_logging("int2lm", launch_time) From 7e4e810f546e0a605a29780fc53d23eefecf36f5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 16 Jan 2024 14:58:25 +0000 Subject: [PATCH 172/245] GitHub Action: Apply Pep8-formatting --- jobs/cosmo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 7018ebba..df94bf2d 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -198,6 +198,6 @@ def main(cfg): logfile_finish=cfg.logfile_finish)) # Submit job - cfg.submit('cosmo', script) + cfg.submit('cosmo', script) cfg.finish_time_logging("cosmo", launch_time) From d615f736ecf5bf9ade77e01534f076c326957e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 16 Jan 2024 15:59:20 +0100 Subject: [PATCH 173/245] Small fixes for int2lm and cosmo --- jobs/cosmo.py | 2 +- jobs/int2lm.py | 2 +- jobs/obs_nudging.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 7018ebba..95a5d52f 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -188,7 +188,7 @@ def main(cfg): with open(runscript_file) as input_file: cosmo_runscript = input_file.read() - script = (cfg.int2lm_work / 'run_cosmo.job') + script = (cfg.cosmo_run / 'run_cosmo.job') with open(script, "w") as outf: outf.write( cosmo_runscript.format(cfg=cfg, diff --git a/jobs/int2lm.py b/jobs/int2lm.py index f185e4e5..d0487fc5 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -143,7 +143,7 @@ def main(cfg): cfg.int2lm['runjob_filename'])) as input_file: int2lm_runscript = input_file.read() - script = (cfg.int2lm_work / 'run_int2lm.job') + script = (cfg.int2lm_run / 'run_int2lm.job') with open(script, "w") as outf: outf.write( int2lm_runscript.format(cfg=cfg, diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index 1bd7c98a..0e86c912 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -3,7 +3,6 @@ import os import logging -import shutil from datetime import timedelta from . import tools From a8b1c090242778ed9592241614aef9e39f68cf45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 12:46:51 +0100 Subject: [PATCH 174/245] Store job scripts in separate directory --- config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index 4c4f3d1c..1da58eb4 100644 --- a/config.py +++ b/config.py @@ -488,7 +488,9 @@ def create_sbatch_script(self, job_name): '', ] - job_file = self.chain_root / f'{job_name}.sh' + job_path = self.chain_root / 'job_scripts' + job_path.mkdir(parents=True, exist_ok=True) + job_file = job_path / f'{job_name}.sh' with open(job_file, mode='w') as job_script: job_script.write('\n'.join(script_lines)) From 763efee782563f59c1ecf1f40fdca7f93959d608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 12:47:29 +0100 Subject: [PATCH 175/245] Add BASIC_PYTHON_JOB to int2lm --- jobs/int2lm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index d0487fc5..aaa714a4 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -9,6 +9,8 @@ from datetime import datetime from . import tools, prepare_cosmo +BASIC_PYTHON_JOB = True + def set_cfg_variables(cfg): cfg.int2lm_run = cfg.chain_root / 'int2lm' / 'run' From 089ec31277fb18463886e19cea23d10a0bdbb157 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 18 Jan 2024 11:48:05 +0000 Subject: [PATCH 176/245] GitHub Action: Apply Pep8-formatting --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 1da58eb4..2dedd30f 100644 --- a/config.py +++ b/config.py @@ -488,7 +488,7 @@ def create_sbatch_script(self, job_name): '', ] - job_path = self.chain_root / 'job_scripts' + job_path = self.chain_root / 'job_scripts' job_path.mkdir(parents=True, exist_ok=True) job_file = job_path / f'{job_name}.sh' with open(job_file, mode='w') as job_script: From 7800847bffcf11bee4fd2c214ded3155ed9c5a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 12:49:32 +0100 Subject: [PATCH 177/245] Add post_cosmo dependency --- workflows.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workflows.yaml b/workflows.yaml index d2fd09dc..8ec87076 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -51,6 +51,9 @@ cosmo-ghg: - post_int2lm previous: - cosmo + post_cosmo: + current: + - cosmo cosmo-art: features: From 019f8f7de644b3b2a2d3e398192e57e65d07d829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 12:58:16 +0100 Subject: [PATCH 178/245] Bugfixes for cosmo jobs --- jobs/biofluxes.py | 5 +++-- jobs/cosmo.py | 5 +++-- jobs/emissions.py | 4 ++-- jobs/int2lm.py | 10 ++-------- jobs/online_vprm.py | 2 +- jobs/post_cosmo.py | 5 ++--- jobs/post_int2lm.py | 5 +++-- jobs/prepare_cosmo.py | 3 +++ 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index 2fde767b..eb0063d0 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -3,7 +3,8 @@ import os import logging -from . import tools, prepare_icon + +from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True @@ -19,8 +20,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("biofluxes") scratch_path = os.path.join(cfg.int2lm_input, 'vprm') diff --git a/jobs/cosmo.py b/jobs/cosmo.py index f091ca8a..6acc75db 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -4,9 +4,10 @@ import logging import os import subprocess + +from datetime import datetime from .tools import write_cosmo_input_ghg from . import tools, prepare_cosmo -from datetime import datetime, timedelta BASIC_PYTHON_JOB = True @@ -40,8 +41,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_cosmo.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("cosmo") logging.info("Setup the namelist for a COSMO tracer run and " diff --git a/jobs/emissions.py b/jobs/emissions.py index f99e8fd1..2e20d569 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -4,7 +4,7 @@ import os import logging -from . import tools, prepare_icon +from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True @@ -32,8 +32,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("emissions") dest_prefix = "emis_" diff --git a/jobs/int2lm.py b/jobs/int2lm.py index aaa714a4..73b85060 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -4,19 +4,14 @@ import os import logging import shutil -import subprocess import pytz + from datetime import datetime from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True -def set_cfg_variables(cfg): - cfg.int2lm_run = cfg.chain_root / 'int2lm' / 'run' - cfg.int2lm_output = cfg.chain_root / 'int2lm' / 'output' - - def main(cfg): """Setup the namelist for int2lm and submit the job to the queue. @@ -48,9 +43,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_cosmo.set_cfg_variables(cfg) - set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("int2lm") # Total number of processes diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 7808ad69..41a6dea5 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -18,8 +18,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - prepare_cosmo.set_cfg_variables() tools.change_logfile(cfg.logfile) + prepare_cosmo.set_cfg_variables() launch_time = cfg.init_time_logging("online_vprm") dest_modis = 'modis.nc' dest_vegetation = 'vegetation.nc' diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 533f8f54..b185e5a3 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -6,7 +6,7 @@ import datetime from subprocess import call -from . import tools, int2lm, cosmo +from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True @@ -65,9 +65,8 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - int2lm.set_cfg_variables(cfg) - cosmo.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("post_cosmo") runscript_path = os.path.join(cfg.cosmo_run, "post_cosmo.job") diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 5823b423..3f6686ca 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -5,8 +5,9 @@ import os import glob import netCDF4 as nc + from datetime import datetime, timedelta -from . import tools, int2lm +from . import tools, prepare_cosmo BASIC_PYTHON_JOB = True @@ -30,7 +31,7 @@ def main(cfg): cfg : Config Object holding all user-configuration parameters as attributes. """ - int2lm.set_cfg_variables(cfg) + prepare_cosmo.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) launch_time = cfg.init_time_logging("post_int2lm") diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 155d9d4d..3718852a 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -13,6 +13,9 @@ def set_cfg_variables(cfg): cfg.int2lm_root = cfg.chain_root / 'int2lm' cfg.int2lm_input = cfg.int2lm_root / 'input' + cfg.int2lm_run = cfg.chain_root / 'int2lm' / 'run' + cfg.int2lm_output = cfg.chain_root / 'int2lm' / 'output' + cfg.cosmo_base = cfg.chain_root / 'cosmo' cfg.cosmo_input = cfg.chain_root / 'cosmo' / 'input' cfg.cosmo_run = cfg.chain_root / 'cosmo' / 'run' From 21d32561bf40974d02b5541acbbc42b662d297e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 13:12:51 +0100 Subject: [PATCH 179/245] Further fixes --- jobs/online_vprm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 41a6dea5..8fe49268 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -19,7 +19,7 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.change_logfile(cfg.logfile) - prepare_cosmo.set_cfg_variables() + prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("online_vprm") dest_modis = 'modis.nc' dest_vegetation = 'vegetation.nc' From 4f87ba3abcc2be17255ecf4955fbe366e328ac22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 15:28:21 +0100 Subject: [PATCH 180/245] Make post_cosmo a submit job --- jobs/post_cosmo.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index b185e5a3..0f07aefb 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -8,7 +8,7 @@ from . import tools, prepare_cosmo -BASIC_PYTHON_JOB = True +BASIC_PYTHON_JOB = False def logfile_header_template(): @@ -69,7 +69,6 @@ def main(cfg): prepare_cosmo.set_cfg_variables(cfg) launch_time = cfg.init_time_logging("post_cosmo") - runscript_path = os.path.join(cfg.cosmo_run, "post_cosmo.job") copy_path = os.path.join( cfg.post_cosmo['output_root'], cfg.startdate_sim_yyyymmddhh + "_" + cfg.enddate_sim_yyyymmddhh) @@ -116,29 +115,18 @@ def main(cfg): logs_src=str(cfg.log_finished_dir).rstrip('/'), logs_dest=logs_path.rstrip('/')) - # Wait for Cosmo to finish first - tools.check_job_completion(cfg.log_finished_dir, "cosmo") + if not cfg.is_async: + # Wait for Cosmo to finish first + tools.check_job_completion(cfg.log_finished_dir, "cosmo") - with open(runscript_path, "w") as script: - script.write(runscript_content) + script = (cfg.cosmo_run / 'run_post_cosmo.job') + with open(script, "w") as outf: + outf.write(runscript_content) logging.info("Submitting the copy job to the xfer queue") logging.info("Make sure you have the module 'xalt' unloaded!") - sbatch_wait = getattr(cfg, "wait", "True") - - if sbatch_wait: - exitcode = call(["sbatch", "--wait", runscript_path]) - logging.info(logfile_header_template().format( - "ENDS", str(datetime.datetime.today()))) - - # copy own logfile aswell - tools.copy_file(cfg.logfile, os.path.join(copy_path, "logs/")) - - else: - exitcode = call(["sbatch", runscript_path]) - - if exitcode != 0: - raise RuntimeError("sbatch returned exitcode {}".format(exitcode)) + # Submit job + cfg.submit('post_cosmo', script) cfg.finish_time_logging("post_cosmo", launch_time) From 4e79263c75280bea71b37ad4fc24244c8472fd95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 15:40:07 +0100 Subject: [PATCH 181/245] Fix post_cosmo --- jobs/cosmo.py | 2 ++ jobs/post_cosmo.py | 1 + 2 files changed, 3 insertions(+) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 6acc75db..c8c24a74 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -4,6 +4,7 @@ import logging import os import subprocess +import pathlib from datetime import datetime from .tools import write_cosmo_input_ghg @@ -189,6 +190,7 @@ def main(cfg): with open(runscript_file) as input_file: cosmo_runscript = input_file.read() + Path(cfg.cosmo_run).mkdir(parents=True, exist_ok=True) script = (cfg.cosmo_run / 'run_cosmo.job') with open(script, "w") as outf: outf.write( diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 0f07aefb..cde9953e 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -119,6 +119,7 @@ def main(cfg): # Wait for Cosmo to finish first tools.check_job_completion(cfg.log_finished_dir, "cosmo") + os.makedirs(cfg.cosmo_run) script = (cfg.cosmo_run / 'run_post_cosmo.job') with open(script, "w") as outf: outf.write(runscript_content) From 58727f494f9b6fbb842ce62c0c91d62ea8777846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Thu, 18 Jan 2024 15:41:04 +0100 Subject: [PATCH 182/245] Fix post_cosmo --- jobs/post_cosmo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index cde9953e..e8f5a600 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -119,7 +119,7 @@ def main(cfg): # Wait for Cosmo to finish first tools.check_job_completion(cfg.log_finished_dir, "cosmo") - os.makedirs(cfg.cosmo_run) + os.makedirs(cfg.cosmo_run, exist_ok=True) script = (cfg.cosmo_run / 'run_post_cosmo.job') with open(script, "w") as outf: outf.write(runscript_content) From 5ebefb5ba20a5e33da6948aebb58581b8fe24066 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Thu, 25 Jan 2024 15:19:34 +0100 Subject: [PATCH 183/245] ref: refactor cycling and monitoring - Regroup waiting, monitoring nd cycling in a single `Config.cycle()` method. - write chunk monitoring info into chain log file --- config.py | 66 +++++++++++++++++++++++++++++++++++++++++----------- run_chain.py | 17 +++----------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/config.py b/config.py index 4c4f3d1c..0773ef63 100644 --- a/config.py +++ b/config.py @@ -59,6 +59,7 @@ def __init__(self, casename): # Set case root self.case_root = self.work_root / self.casename + self.log_file = self.case_root / "chain_status.log" # Set workflow and async attributes and initiate job ids dict self.set_workflow() @@ -337,12 +338,11 @@ def log_job_status(self, job, status, launch_time, duration=None): - log_job_status('chain', 'FINISH', datetime.datetime.now(), '00:15:30') - log_job_status('task_1', 'ERROR', datetime.datetime(2023, 12, 20, 8, 30, 15)) """ - log_file = self.case_root / "chain_status.log" # Check if the file exists, if not, create it and write header - if not log_file.is_file(): + if not self.log_file.is_file(): header = "Name ID Status Time Duration\n" - with open(log_file, 'w') as f: + with self.log_file.open('w') as f: f.write(header) # Format duration and chunk_id @@ -362,7 +362,7 @@ def log_job_status(self, job, status, launch_time, duration=None): else: log_entry = f"{job:<15} {chunk_id:<21} {status:<6} {launch_time:<24}\n" - with open(log_file, 'a') as f: + with self.log_file.open('a') as f: f.write(log_entry) def format_duration(self, duration): @@ -522,6 +522,43 @@ def wait_for_previous(self): subprocess.run(['sbatch', '--wait', job_file], check=True) + def cycle(self): + """Cycle to next chunk + + - Wait for previous chunk to finish + - print summary of previous chunk jobs + - Check for success of all previous jobs + - cycle job ids and chunk id""" + + # - ML - + # - This method could do more of the cycling, like dates + # - The config object could host more info and cycle it instead + # of recomputing stuff like previous chunk dates + + # Skip if very first chunk + if cfg.job_ids['previous']: + # Wait for previous chunk to be done + self.wait_for_previous() + + # Get and print previous chunk Slurm summary + self.get_previous_slurm_summary() + self.print_previous_slurm_summary() + + # Check for success of all previous jobs + self.check_previous_chunk_success() + + # Cycle info + self.job_ids['previous'] = self.job_ids['current'] + self.previous_chunk_id = self.chunk_id + + # Monitor last chunk + if cfg.enddate_sim >= cfg.enddate: + self.wait_for_previous() + self.get_previous_slurm_summary() + self.print_previous_slurm_summary() + self.check_previous_chunk_success() + + @staticmethod def get_job_info(job_id, slurm_keys=['JobName', 'Elapsed', 'ExitCode'], @@ -547,7 +584,7 @@ def get_job_info(job_id, else: return info_str.decode() - def get_slurm_summary(self, + def get_previous_slurm_summary(self, info_keys=[ 'JobName', 'JobID', 'Partition', 'NNodes', 'State', 'Start', 'End', 'Elapsed' @@ -566,7 +603,7 @@ def get_slurm_summary(self, self.get_job_info(job_id, slurm_keys=info_keys, parse=True)) - def print_slurm_summary(self): + def print_previous_slurm_summary(self): # Width of printed slurm piece of information info_width = { @@ -593,15 +630,16 @@ def print_slurm_summary(self): table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) line_format = " ".join(formats) - print(" └── Slurm info of previous submitted jobs") - - for job_name in self.jobs: - print(f" └── {job_name}") - print(table_header) - for info in self.slurm_info[job_name]: - print(line_format.format(**info)) + with self.log_file.open('a') as f: + f.write(f"Job summary for chunk {self.previous_chunk_id}") + f.write('') + f.write(table_header) + for job_name in self.jobs: + for info in self.slurm_info[job_name]: + f.write(line_format.format(**info)) + f.write('') - def check_chunk_success(self): + def check_previous_chunk_success(self): status = 0 failed_jobs = [] for job_name, info_list in self.slurm_info.items(): diff --git a/run_chain.py b/run_chain.py index 801b9249..22a4e70f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -249,20 +249,9 @@ def run_chunk(cfg, force, resume): else: job.main(cfg) - # Return if very first chunk - if cfg.job_ids['previous']: - # Wait for previous chunk to be done - cfg.wait_for_previous() - - # Current chunk Slurm summary - cfg.get_slurm_summary() - cfg.print_slurm_summary() - - # Check for success - cfg.check_chunk_success() - - # Cycle - cfg.job_ids['previous'] = cfg.job_ids['current'] + # Wait for previous chunk jobs, monitor them and cycle info + cfg.cycle() + else: # run jobs (if required) for job in cfg.jobs: From f5be54c0d919cc1ad0ace9706bff01304a0a316e Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 25 Jan 2024 14:28:01 +0000 Subject: [PATCH 184/245] GitHub Action: Apply Pep8-formatting --- config.py | 10 +++++----- run_chain.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 9d983d0a..ea534747 100644 --- a/config.py +++ b/config.py @@ -559,7 +559,6 @@ def cycle(self): self.get_previous_slurm_summary() self.print_previous_slurm_summary() self.check_previous_chunk_success() - @staticmethod def get_job_info(job_id, @@ -587,10 +586,11 @@ def get_job_info(job_id, return info_str.decode() def get_previous_slurm_summary(self, - info_keys=[ - 'JobName', 'JobID', 'Partition', 'NNodes', - 'State', 'Start', 'End', 'Elapsed' - ]): + info_keys=[ + 'JobName', 'JobID', 'Partition', + 'NNodes', 'State', 'Start', 'End', + 'Elapsed' + ]): """get slurm info summary or all jobs of previous chunk""" # Store requested keys in object diff --git a/run_chain.py b/run_chain.py index 22a4e70f..5646395e 100755 --- a/run_chain.py +++ b/run_chain.py @@ -251,7 +251,7 @@ def run_chunk(cfg, force, resume): # Wait for previous chunk jobs, monitor them and cycle info cfg.cycle() - + else: # run jobs (if required) for job in cfg.jobs: From 1a3629a6bc04bdd6225ecff041285b52e3c52f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 26 Jan 2024 12:44:28 +0100 Subject: [PATCH 185/245] Fix config variables --- cases/icon-art-global-test/icon_runjob.cfg | 2 +- cases/icon-art-oem-test/icon_runjob.cfg | 2 +- cases/icon-seq-test/icon_runjob.cfg | 2 +- cases/icon-test/icon_runjob.cfg | 2 +- config.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index 2c7db9d3..e5334c64 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -7,7 +7,7 @@ #SBATCH --partition={cfg.compute_queue} #SBATCH --constraint={cfg.constraint} #SBATCH --hint=nomultithread -#SBATCH --output={logfile} +#SBATCH --output={cfg.logfile} #SBATCH --open-mode=append #SBATCH --chdir={cfg.icon_work} diff --git a/cases/icon-art-oem-test/icon_runjob.cfg b/cases/icon-art-oem-test/icon_runjob.cfg index 4b3190b0..d64de85c 100644 --- a/cases/icon-art-oem-test/icon_runjob.cfg +++ b/cases/icon-art-oem-test/icon_runjob.cfg @@ -7,7 +7,7 @@ #SBATCH --partition={cfg.compute_queue} #SBATCH --constraint={cfg.constraint} #SBATCH --hint=nomultithread -#SBATCH --output={logfile} +#SBATCH --output={cfg.logfile} #SBATCH --open-mode=append #SBATCH --chdir={cfg.icon_work} diff --git a/cases/icon-seq-test/icon_runjob.cfg b/cases/icon-seq-test/icon_runjob.cfg index f0ba8453..9fd03776 100755 --- a/cases/icon-seq-test/icon_runjob.cfg +++ b/cases/icon-seq-test/icon_runjob.cfg @@ -7,7 +7,7 @@ #SBATCH --partition={cfg.compute_queue} #SBATCH --constraint={cfg.constraint} #SBATCH --hint=nomultithread -#SBATCH --output={logfile} +#SBATCH --output={cfg.logfile} #SBATCH --open-mode=append #SBATCH --chdir={cfg.icon_work} diff --git a/cases/icon-test/icon_runjob.cfg b/cases/icon-test/icon_runjob.cfg index f0ba8453..9fd03776 100755 --- a/cases/icon-test/icon_runjob.cfg +++ b/cases/icon-test/icon_runjob.cfg @@ -7,7 +7,7 @@ #SBATCH --partition={cfg.compute_queue} #SBATCH --constraint={cfg.constraint} #SBATCH --hint=nomultithread -#SBATCH --output={logfile} +#SBATCH --output={cfg.logfile} #SBATCH --open-mode=append #SBATCH --chdir={cfg.icon_work} diff --git a/config.py b/config.py index ea534747..134b06c0 100644 --- a/config.py +++ b/config.py @@ -538,7 +538,7 @@ def cycle(self): # of recomputing stuff like previous chunk dates # Skip if very first chunk - if cfg.job_ids['previous']: + if self.job_ids['previous']: # Wait for previous chunk to be done self.wait_for_previous() @@ -554,7 +554,7 @@ def cycle(self): self.previous_chunk_id = self.chunk_id # Monitor last chunk - if cfg.enddate_sim >= cfg.enddate: + if self.enddate_sim >= self.enddate: self.wait_for_previous() self.get_previous_slurm_summary() self.print_previous_slurm_summary() From f51a3369dd336e7119c1f513488ce39ea3ce7d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 26 Jan 2024 13:01:18 +0100 Subject: [PATCH 186/245] Fix log file output --- config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 134b06c0..4a9b757a 100644 --- a/config.py +++ b/config.py @@ -606,7 +606,6 @@ def get_previous_slurm_summary(self, parse=True)) def print_previous_slurm_summary(self): - # Width of printed slurm piece of information info_width = { 'JobName': 10, @@ -633,13 +632,12 @@ def print_previous_slurm_summary(self): line_format = " ".join(formats) with self.log_file.open('a') as f: - f.write(f"Job summary for chunk {self.previous_chunk_id}") - f.write('') + f.write(f"Job summary for chunk {self.previous_chunk_id}\n") f.write(table_header) for job_name in self.jobs: for info in self.slurm_info[job_name]: f.write(line_format.format(**info)) - f.write('') + f.write('\n') def check_previous_chunk_success(self): status = 0 From 30a81a0529522fa4c7f17783b80afa2fa9ad2273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 26 Jan 2024 13:40:59 +0100 Subject: [PATCH 187/245] Formatting --- config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.py b/config.py index 4a9b757a..4b2b36ee 100644 --- a/config.py +++ b/config.py @@ -634,10 +634,12 @@ def print_previous_slurm_summary(self): with self.log_file.open('a') as f: f.write(f"Job summary for chunk {self.previous_chunk_id}\n") f.write(table_header) + f.write('\n') for job_name in self.jobs: for info in self.slurm_info[job_name]: f.write(line_format.format(**info)) f.write('\n') + f.write('\n') def check_previous_chunk_success(self): status = 0 From d603c074e358cf64e916daa9e2859050e1add23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 26 Jan 2024 15:44:41 +0100 Subject: [PATCH 188/245] Remove time logging from jobs --- jobs/biofluxes.py | 2 -- jobs/check_output.py | 2 -- jobs/cosmo.py | 2 -- jobs/emissions.py | 2 -- jobs/icon.py | 2 -- jobs/icontools.py | 2 -- jobs/int2lm.py | 2 -- jobs/obs_nudging.py | 2 -- jobs/octe.py | 2 -- jobs/oem.py | 2 -- jobs/online_vprm.py | 2 -- jobs/photo_rate.py | 2 -- jobs/post_cosmo.py | 2 -- jobs/post_int2lm.py | 2 -- jobs/prepare_art.py | 2 -- jobs/prepare_art_global.py | 2 -- jobs/prepare_art_oem.py | 2 -- jobs/prepare_cosmo.py | 2 -- jobs/prepare_icon.py | 2 -- jobs/reduce_output.py | 2 -- jobs/verify_chain.py | 2 -- 21 files changed, 42 deletions(-) diff --git a/jobs/biofluxes.py b/jobs/biofluxes.py index eb0063d0..cc9a738e 100644 --- a/jobs/biofluxes.py +++ b/jobs/biofluxes.py @@ -22,7 +22,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("biofluxes") scratch_path = os.path.join(cfg.int2lm_input, 'vprm') @@ -48,4 +47,3 @@ def main(cfg): logging.error( "Splitting or copying of GPP or/and RA files to scratch failed." ) - cfg.finish_time_logging("biofluxes", launch_time) diff --git a/jobs/check_output.py b/jobs/check_output.py index 22efd12e..e8737a12 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -716,7 +716,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("check_output") date = dt.datetime.today() to_print = """check_output @@ -866,7 +865,6 @@ def main(cfg): raise RuntimeError('Logfile containing errors! See %s' % cfg.logfile) - cfg.finish_time_logging("check_output", launch_time) if __name__ == '__main__': diff --git a/jobs/cosmo.py b/jobs/cosmo.py index c8c24a74..8f8dda77 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -44,7 +44,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("cosmo") logging.info("Setup the namelist for a COSMO tracer run and " "submit the job to the queue") @@ -203,4 +202,3 @@ def main(cfg): # Submit job cfg.submit('cosmo', script) - cfg.finish_time_logging("cosmo", launch_time) diff --git a/jobs/emissions.py b/jobs/emissions.py index 2e20d569..5e18cb79 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -34,7 +34,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("emissions") dest_prefix = "emis_" if not isinstance(cfg.emissions['dir'], list): @@ -72,4 +71,3 @@ def main(cfg): if cfg.workflow_name.startswith('cosmo'): tools.string2char.main(dest_path) - cfg.finish_time_logging("emissions", launch_time) diff --git a/jobs/icon.py b/jobs/icon.py index 0461f3fe..f382c193 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -36,7 +36,6 @@ def main(cfg): prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) if BASIC_PYTHON_JOB: - launch_time = cfg.init_time_logging("icon") logging.info("Setup the namelist for an ICON run and " "submit the job to the queue") @@ -68,4 +67,3 @@ def main(cfg): cfg.submit('icon', script) if BASIC_PYTHON_JOB: - cfg.finish_time_logging("icon", launch_time) diff --git a/jobs/icontools.py b/jobs/icontools.py index ad5006cc..ceaada63 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -21,7 +21,6 @@ def main(cfg): """ prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("icontools") #----------------------------------------------------- # Create LBC datafile lists (each at 00 UTC and others) @@ -109,4 +108,3 @@ def main(cfg): tools.rename_file(merged_file, src_file) logging.info('OK') - cfg.finish_time_logging("icontools", launch_time) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index 73b85060..c17be708 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -45,7 +45,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("int2lm") # Total number of processes np_tot = cfg.int2lm['np_x'] * cfg.int2lm['np_y'] @@ -155,4 +154,3 @@ def main(cfg): # Submit job cfg.submit('int2lm', script) - cfg.finish_time_logging("int2lm", launch_time) diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index 0e86c912..bce94e12 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -29,7 +29,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("obs_nudging") dest_dir = os.path.join(cfg.cosmo_input, "obs_nudging") tools.create_dir(dest_dir, "obs nudging input") @@ -60,4 +59,3 @@ def main(cfg): logging.info("Copied blacklist-file to {}".format(dest_dir)) - cfg.finish_time_logging("obs_nudging", launch_time) diff --git a/jobs/octe.py b/jobs/octe.py index 058853d4..1efbe486 100644 --- a/jobs/octe.py +++ b/jobs/octe.py @@ -180,7 +180,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("octe") dest_dir = join(cfg.cosmo_input, 'octe') create_dir_and_copy_input(dest_dir=dest_dir, lambdas_src=cfg.octe_lambdas, @@ -191,4 +190,3 @@ def main(cfg): logging.info("Starting to create BG-ensembles in " + cfg.int2lm_output) perturb_bgs_in_dir(cfg.octe_lambdas, cfg.int2lm_output) logging.info("Finished creating BG-ensembles") - cfg.finish_time_logging("octe", launch_time) diff --git a/jobs/oem.py b/jobs/oem.py index 64d93dc8..66050820 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -25,7 +25,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("oem") oem_dir = cfg.oem['dir'] oem_gridded_emissions_nc = os.path.join(oem_dir, @@ -87,4 +86,3 @@ def main(cfg): tools.copy_file(os.path.join(oem_dir, cfg.oem['ens_lambda_nc']), os.path.join(dest_dir, cfg.oem['ens_lambda_nc'])) - cfg.finish_time_logging("oem", launch_time) diff --git a/jobs/online_vprm.py b/jobs/online_vprm.py index 8fe49268..a1e2312e 100644 --- a/jobs/online_vprm.py +++ b/jobs/online_vprm.py @@ -20,7 +20,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("online_vprm") dest_modis = 'modis.nc' dest_vegetation = 'vegetation.nc' @@ -38,4 +37,3 @@ def main(cfg): src_dir, dest_dir)) tools.copy_file(vegetation_data_nc, os.path.join(dest_dir, dest_vegetation)) - cfg.finish_time_logging("online_vprm", launch_time) diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index 0814b753..c7e7120f 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -23,7 +23,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes. """ tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("photo_rate") logging.info("Copying photolysis-rate file from {} to {}".format( cfg.photo_rate_file, @@ -39,4 +38,3 @@ def main(cfg): logging.info("Finished") - cfg.finish_time_logging("photo_rate", launch_time) diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index e8f5a600..78262328 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -67,7 +67,6 @@ def main(cfg): """ tools.change_logfile(cfg.logfile) prepare_cosmo.set_cfg_variables(cfg) - launch_time = cfg.init_time_logging("post_cosmo") copy_path = os.path.join( cfg.post_cosmo['output_root'], @@ -130,4 +129,3 @@ def main(cfg): # Submit job cfg.submit('post_cosmo', script) - cfg.finish_time_logging("post_cosmo", launch_time) diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index 3f6686ca..e0056b30 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -33,7 +33,6 @@ def main(cfg): """ prepare_cosmo.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("post_int2lm") # Int2lm processing always starts at hstart=0, thus modifying inidate inidate_int2lm_yyyymmddhh = cfg.startdate_sim_yyyymmddhh @@ -157,4 +156,3 @@ def main(cfg): logging.info(to_print) - cfg.finish_time_logging("post_int2lm", launch_time) diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index 0f120fc2..e7f34003 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -19,7 +19,6 @@ def main(cfg): """ prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("prepare_art") logging.info('Add Q (copy of QV) and/or PS to initial file') meteo_file = os.path.join( @@ -55,4 +54,3 @@ def main(cfg): tools.rename_file(merged_file, meteo_file) logging.info('OK') - cfg.finish_time_logging("prepare_art", launch_time) diff --git a/jobs/prepare_art_global.py b/jobs/prepare_art_global.py index 9ceac550..a425d6b9 100644 --- a/jobs/prepare_art_global.py +++ b/jobs/prepare_art_global.py @@ -25,7 +25,6 @@ def main(cfg): """ prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("prepare_art_global") logging.info("Prepare ICON-ART for global simulations") # -- Download ERA5 data and create the inicond file @@ -179,4 +178,3 @@ def main(cfg): process.communicate() logging.info("OK") - cfg.finish_time_logging("prepare_art_global", launch_time) diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 5432efb6..2f0e86bd 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -15,7 +15,6 @@ def main(cfg): """ prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("prepare_art") logging.info('Merging IC and LBC') if cfg.input_files['oem_gridded_emissions_nc']: @@ -86,4 +85,3 @@ def main(cfg): logging.info( "Added chemical tracer to file {}".format(merged_file)) - cfg.finish_time_logging("prepare_art_oem", launch_time) diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 3718852a..a55bd9ac 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -78,7 +78,6 @@ def main(cfg): """ set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("prepare_cosmo") logging.info('COSMO analysis data for IC/BC') @@ -236,4 +235,3 @@ def main(cfg): logging.info("OK") - cfg.finish_time_logging("prepare_cosmo", launch_time) diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 7c2f1ca3..58241dc6 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -68,7 +68,6 @@ def main(cfg): """ set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("prepare_icon") # Create directories tools.create_dir(cfg.icon_work, "icon_work") @@ -97,4 +96,3 @@ def main(cfg): cfg.submit('prepare_icon', script) logging.info("OK") - cfg.finish_time_logging("prepare_icon", launch_time) diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index b62a97a1..ca359b22 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -45,7 +45,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes """ tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("reduce_output") cosmo_output = cfg.cosmo_output output_path = cfg.cosmo_output_reduced @@ -163,4 +162,3 @@ def main(cfg): logging.info(to_print) - cfg.finish_time_logging("reduce_output", launch_time) diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 95f6a712..20f9b01b 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -41,7 +41,6 @@ def main(cfg): Object holding all user-configuration parameters as attributes """ tools.change_logfile(cfg.logfile) - launch_time = cfg.init_time_logging("verify_chain") logging.info("Started verification") for (ref_file, run_file), variables in cfg.verify_chain['values_to_check'].items(): @@ -72,4 +71,3 @@ def main(cfg): logging.info("Finished verification") - cfg.finish_time_logging("verify_chain", launch_time) From 532af08b5a43a7e896b97ae06f4530845a5fe0e1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 26 Jan 2024 14:45:01 +0000 Subject: [PATCH 189/245] GitHub Action: Apply Pep8-formatting --- jobs/post_int2lm.py | 1 - jobs/prepare_art_oem.py | 1 - 2 files changed, 2 deletions(-) diff --git a/jobs/post_int2lm.py b/jobs/post_int2lm.py index e0056b30..b0e78c0c 100644 --- a/jobs/post_int2lm.py +++ b/jobs/post_int2lm.py @@ -155,4 +155,3 @@ def main(cfg): =====================================================""" % date.strftime('%s') logging.info(to_print) - diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 2f0e86bd..8e4abe17 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -84,4 +84,3 @@ def main(cfg): tools.remove_file(chem_file) logging.info( "Added chemical tracer to file {}".format(merged_file)) - From 70204ca4bfbd3d7b6515403177f93470325ee77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Fri, 26 Jan 2024 15:49:52 +0100 Subject: [PATCH 190/245] Fix for icon job --- jobs/icon.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jobs/icon.py b/jobs/icon.py index f382c193..468def85 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -35,7 +35,6 @@ def main(cfg): """ prepare_icon.set_cfg_variables(cfg) tools.change_logfile(cfg.logfile) - if BASIC_PYTHON_JOB: logging.info("Setup the namelist for an ICON run and " "submit the job to the queue") @@ -66,4 +65,3 @@ def main(cfg): # Submit run script cfg.submit('icon', script) - if BASIC_PYTHON_JOB: From 58fd928f5a20cae8fce3718537dfc611f10db601 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 26 Jan 2024 14:50:27 +0000 Subject: [PATCH 191/245] GitHub Action: Apply Pep8-formatting --- jobs/check_output.py | 1 - jobs/cosmo.py | 1 - jobs/emissions.py | 1 - jobs/icon.py | 1 - jobs/icontools.py | 1 - jobs/int2lm.py | 1 - jobs/obs_nudging.py | 1 - jobs/oem.py | 1 - jobs/photo_rate.py | 1 - jobs/post_cosmo.py | 1 - jobs/prepare_art.py | 1 - jobs/prepare_cosmo.py | 1 - jobs/prepare_icon.py | 1 - jobs/reduce_output.py | 1 - jobs/verify_chain.py | 1 - 15 files changed, 15 deletions(-) diff --git a/jobs/check_output.py b/jobs/check_output.py index e8737a12..5ab99614 100644 --- a/jobs/check_output.py +++ b/jobs/check_output.py @@ -866,7 +866,6 @@ def main(cfg): cfg.logfile) - if __name__ == '__main__': args = sys.argv diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 8f8dda77..fbb62d74 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -201,4 +201,3 @@ def main(cfg): # Submit job cfg.submit('cosmo', script) - diff --git a/jobs/emissions.py b/jobs/emissions.py index 5e18cb79..0b252c4c 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -70,4 +70,3 @@ def main(cfg): # (NF90_CHAR) (needed for int2lm to work) if cfg.workflow_name.startswith('cosmo'): tools.string2char.main(dest_path) - diff --git a/jobs/icon.py b/jobs/icon.py index 468def85..393f740e 100644 --- a/jobs/icon.py +++ b/jobs/icon.py @@ -64,4 +64,3 @@ def main(cfg): # Submit run script cfg.submit('icon', script) - diff --git a/jobs/icontools.py b/jobs/icontools.py index ceaada63..c68dc485 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -107,4 +107,3 @@ def main(cfg): # Rename file to get original file name tools.rename_file(merged_file, src_file) logging.info('OK') - diff --git a/jobs/int2lm.py b/jobs/int2lm.py index c17be708..c79169d8 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -153,4 +153,3 @@ def main(cfg): # Submit job cfg.submit('int2lm', script) - diff --git a/jobs/obs_nudging.py b/jobs/obs_nudging.py index bce94e12..57119af1 100644 --- a/jobs/obs_nudging.py +++ b/jobs/obs_nudging.py @@ -58,4 +58,3 @@ def main(cfg): tools.copy_file(os.path.join(cfg.obs_nudging_dir, 'blklsttmp'), dest_dir) logging.info("Copied blacklist-file to {}".format(dest_dir)) - diff --git a/jobs/oem.py b/jobs/oem.py index 66050820..8ad1511e 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -85,4 +85,3 @@ def main(cfg): if cfg.oem.get('oem_ens_lambda_nc') is not None: tools.copy_file(os.path.join(oem_dir, cfg.oem['ens_lambda_nc']), os.path.join(dest_dir, cfg.oem['ens_lambda_nc'])) - diff --git a/jobs/photo_rate.py b/jobs/photo_rate.py index c7e7120f..afdb4716 100644 --- a/jobs/photo_rate.py +++ b/jobs/photo_rate.py @@ -37,4 +37,3 @@ def main(cfg): tools.copy_file(src_file, dest_path) logging.info("Finished") - diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 78262328..13fa6e7b 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -128,4 +128,3 @@ def main(cfg): # Submit job cfg.submit('post_cosmo', script) - diff --git a/jobs/prepare_art.py b/jobs/prepare_art.py index e7f34003..749fd7ce 100644 --- a/jobs/prepare_art.py +++ b/jobs/prepare_art.py @@ -53,4 +53,3 @@ def main(cfg): ds.to_netcdf(merged_file) tools.rename_file(merged_file, meteo_file) logging.info('OK') - diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index a55bd9ac..444d78e8 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -234,4 +234,3 @@ def main(cfg): tools.copy_file(filename, scratch_path, output_log=True) logging.info("OK") - diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index 58241dc6..f6c6ac1b 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -95,4 +95,3 @@ def main(cfg): cfg.submit('prepare_icon', script) logging.info("OK") - diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index ca359b22..7163b904 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -161,4 +161,3 @@ def main(cfg): =====================================================""" % date.strftime("%s") logging.info(to_print) - diff --git a/jobs/verify_chain.py b/jobs/verify_chain.py index 20f9b01b..05b537a9 100644 --- a/jobs/verify_chain.py +++ b/jobs/verify_chain.py @@ -70,4 +70,3 @@ def main(cfg): comp_data(ref_data, run_data, variables) logging.info("Finished verification") - From 8810dcf3e42c5ae38f5f4e6682d217fd0c5d2d32 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 29 Jan 2024 09:55:25 +0100 Subject: [PATCH 192/245] ref: exception handling in Config.submit() --- config.py | 26 +++++++++++--------------- jobs/post_cosmo.py | 1 - 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 4b2b36ee..fc2eae34 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,4 @@ -import subprocess +from subprocess import run, CalledProcessError import os import yaml @@ -445,9 +445,14 @@ def submit(self, job_name, script, add_dep=None, logfile=None): sbatch_cmd.append(dep_cmd) sbatch_cmd.append(script_path.name) - result = subprocess.run(sbatch_cmd, - cwd=script_path.parent, - capture_output=True) + try: + result = run(sbatch_cmd, cwd=script_path.parent, + capture_output=True, check=True) + except CalledProcessError as e: + with open self.logfile('a') as f: + f.write(e) + raise(e) + job_id = int(result.stdout) print(f' └── Submitted batch job {job_id}') @@ -456,17 +461,8 @@ def submit(self, job_name, script, add_dep=None, logfile=None): else: self.job_ids['current'][job_name].append(job_id) - exitcode = result.returncode - self.check_job(exitcode, logfile=logfile) - return job_id - def check_job(self, exitcode, logfile=None): - """Check the exitcode returned by a job.""" - - if exitcode != 0: - raise RuntimeError(f"sbatch returned exitcode {exitcode}") - def create_sbatch_script(self, job_name): """Create an sbatch script to launch jobs individually. Use run_chain.py arguments to submit those jobs. @@ -522,7 +518,7 @@ def wait_for_previous(self): with open(job_file, mode='w') as wait_job: wait_job.write('\n'.join(script_lines)) - subprocess.run(['sbatch', '--wait', job_file], check=True) + run(['sbatch', '--wait', job_file], check=True) def cycle(self): """Cycle to next chunk @@ -575,7 +571,7 @@ def get_job_info(job_id, if parse: cmd.append("--parsable") - info_str = subprocess.run(cmd, capture_output=True, check=True).stdout + info_str = run(cmd, capture_output=True, check=True).stdout if parse: # Parse in a dictionnary before returning diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 13fa6e7b..22df632d 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -4,7 +4,6 @@ import logging import os import datetime -from subprocess import call from . import tools, prepare_cosmo From 1c4d9f027690c1cea0908858abf0a1abf085e4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 11:28:50 +0100 Subject: [PATCH 193/245] Change table cell widths --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 4c4f3d1c..1cbbb818 100644 --- a/config.py +++ b/config.py @@ -570,10 +570,10 @@ def print_slurm_summary(self): # Width of printed slurm piece of information info_width = { - 'JobName': 10, + 'JobName': 13, 'JobID': 8, 'Partition': 9, - 'NNodes': 6, + 'N': 3, 'State': 14, 'Start': 13, 'End': 13, From f036e43f0085e7183930c2d6348364a43f24fa0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 11:32:18 +0100 Subject: [PATCH 194/245] Remove old way of logging --- config.py | 58 ---------------------------------------------------- run_chain.py | 9 -------- 2 files changed, 67 deletions(-) diff --git a/config.py b/config.py index 1cbbb818..de64edc8 100644 --- a/config.py +++ b/config.py @@ -318,53 +318,6 @@ def create_vars_from_dicts(self, dct=None, key=None): else: setattr(self, subkey, v) - def log_job_status(self, job, status, launch_time, duration=None): - """ - Log the status of a job in a chain to a file. - - Parameters: - - job (str): The type of job, either 'chain' or a specific job name. - - status (str): The status of the job, e.g., 'FINISH', 'ERROR', etc. - - launch_time (datetime.datetime): The timestamp when the job was launched or finished. - - duration (datetime.timedelta, optional): The duration of the job. Default is None. - - The function logs the job information to a file named 'chain_status.log' in the case root directory. - If the log file doesn't exist, it creates a header with the column names. - The logged entry includes the job type, job ID (if applicable), status, launch time, - and optionally, the duration if provided. - - Example: - - log_job_status('chain', 'FINISH', datetime.datetime.now(), '00:15:30') - - log_job_status('task_1', 'ERROR', datetime.datetime(2023, 12, 20, 8, 30, 15)) - """ - log_file = self.case_root / "chain_status.log" - - # Check if the file exists, if not, create it and write header - if not log_file.is_file(): - header = "Name ID Status Time Duration\n" - with open(log_file, 'w') as f: - f.write(header) - - # Format duration and chunk_id - if job == 'chain': - if duration is not None: - duration = self.format_duration(duration) - chunk_id = self.casename - else: - if duration is not None: - duration = f"{str(int(duration.total_seconds()))} s" - chunk_id = self.chunk_id - - # Log the job information - launch_time = launch_time.strftime("%a %b %d %Y %H:%M:%S") - if status == 'FINISH' and duration: - log_entry = f"{job:<15} {chunk_id:<21} {status:<6} {launch_time:<24} {duration}\n" - else: - log_entry = f"{job:<15} {chunk_id:<21} {status:<6} {launch_time:<24}\n" - - with open(log_file, 'a') as f: - f.write(log_entry) - def format_duration(self, duration): """ Format a duration represented by a datetime.timedelta object into a human-readable string. @@ -383,17 +336,6 @@ def format_duration(self, duration): formatted_duration = f"{int(days)}d {int(hours)}h {int(minutes)}m {int(seconds)}s" return formatted_duration - def init_time_logging(self, job): - launch_time = datetime.now() - self.log_job_status(job, 'START', launch_time) - - return launch_time - - def finish_time_logging(self, job, launch_time): - end_time = datetime.now() - duration = end_time - launch_time - self.log_job_status(job, 'FINISH', end_time, duration) - def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" # Initial list of dependencies diff --git a/run_chain.py b/run_chain.py index 801b9249..838528a4 100755 --- a/run_chain.py +++ b/run_chain.py @@ -470,9 +470,6 @@ def main(): # Make ntry a Config variable cfg.ntry = args.ntry - # Check logging settings - cfg.logging = args.enable_logging - # Convert relative to absolute paths cfg.convert_paths_to_absolute() @@ -513,9 +510,6 @@ def main(): else: print("Running the Processing Chain in sequential mode.") - if cfg.logging: - launch_time = cfg.init_time_logging('chain') - # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: if hasattr(cfg, 'spinup'): @@ -532,9 +526,6 @@ def main(): cfg.enddate_sim = cfg.enddate run_chunk(cfg=cfg, force=args.force, resume=args.resume) - if cfg.logging: - cfg.finish_time_logging('chain', launch_time) - print('>>> Finished the processing chain successfully <<<') From 19d45b03b378e4e9d1d1910bca68e673177f34e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 11:43:04 +0100 Subject: [PATCH 195/245] Cleanup --- config.py | 9 ++++----- jobs/cosmo.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 4abd39c2..b00d70ec 100644 --- a/config.py +++ b/config.py @@ -4,7 +4,6 @@ from jobs import tools from pathlib import Path -from datetime import datetime class Config(): @@ -566,10 +565,10 @@ def print_previous_slurm_summary(self): hlines = [] formats = [] for k in self.info_keys: - l = info_width[k] - formats.append(f"{{{k}:>{l}.{l}}}") - headers.append(f"{k:>{l}.{l}}") - hlines.append("-" * l) + j = info_width[k] + formats.append(f"{{{k}:>{j}.{j}}}") + headers.append(f"{k:>{j}.{j}}") + hlines.append("-" * j) table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) line_format = " ".join(formats) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index fbb62d74..3ebfa341 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -4,7 +4,7 @@ import logging import os import subprocess -import pathlib +from pathlib import Path from datetime import datetime from .tools import write_cosmo_input_ghg From fb19c02d2a152de4c51af563dd27a8a6ee90b41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 11:44:29 +0100 Subject: [PATCH 196/245] NNodes -> N --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index b00d70ec..2455d398 100644 --- a/config.py +++ b/config.py @@ -530,7 +530,7 @@ def get_job_info(job_id, def get_previous_slurm_summary(self, info_keys=[ 'JobName', 'JobID', 'Partition', - 'NNodes', 'State', 'Start', 'End', + 'N', 'State', 'Start', 'End', 'Elapsed' ]): """get slurm info summary or all jobs of previous chunk""" From 7b618ee8205f4f9bed1e33a425d22713fc8057a5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 29 Jan 2024 10:44:44 +0000 Subject: [PATCH 197/245] GitHub Action: Apply Pep8-formatting --- config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 2455d398..4f11c61b 100644 --- a/config.py +++ b/config.py @@ -529,9 +529,8 @@ def get_job_info(job_id, def get_previous_slurm_summary(self, info_keys=[ - 'JobName', 'JobID', 'Partition', - 'N', 'State', 'Start', 'End', - 'Elapsed' + 'JobName', 'JobID', 'Partition', 'N', + 'State', 'Start', 'End', 'Elapsed' ]): """get slurm info summary or all jobs of previous chunk""" From 5a314f4b659bd24f9568676ade3148bce66bd1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 12:04:30 +0100 Subject: [PATCH 198/245] Fix for N/NNodes --- config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 4f11c61b..d5517817 100644 --- a/config.py +++ b/config.py @@ -529,7 +529,7 @@ def get_job_info(job_id, def get_previous_slurm_summary(self, info_keys=[ - 'JobName', 'JobID', 'Partition', 'N', + 'JobName', 'JobID', 'Partition', 'NNodes', 'State', 'Start', 'End', 'Elapsed' ]): """get slurm info summary or all jobs of previous chunk""" @@ -552,7 +552,7 @@ def print_previous_slurm_summary(self): 'JobName': 13, 'JobID': 8, 'Partition': 9, - 'N': 3, + 'NNodes': 3, 'State': 14, 'Start': 13, 'End': 13, @@ -565,6 +565,7 @@ def print_previous_slurm_summary(self): formats = [] for k in self.info_keys: j = info_width[k] + k = '#N' if k == 'NNodes' else k formats.append(f"{{{k}:>{j}.{j}}}") headers.append(f"{k:>{j}.{j}}") hlines.append("-" * j) From 7933fb4c827c19d628561b6f7d0c90ef5031b4eb Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 29 Jan 2024 11:04:53 +0000 Subject: [PATCH 199/245] GitHub Action: Apply Pep8-formatting --- config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index d5517817..c0a09073 100644 --- a/config.py +++ b/config.py @@ -529,8 +529,9 @@ def get_job_info(job_id, def get_previous_slurm_summary(self, info_keys=[ - 'JobName', 'JobID', 'Partition', 'NNodes', - 'State', 'Start', 'End', 'Elapsed' + 'JobName', 'JobID', 'Partition', + 'NNodes', 'State', 'Start', 'End', + 'Elapsed' ]): """get slurm info summary or all jobs of previous chunk""" From c4f30ae85a40c4022b2588334e9c467bef4cbce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 12:13:50 +0100 Subject: [PATCH 200/245] Yet another fix for N/NNodes --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c0a09073..06f43d42 100644 --- a/config.py +++ b/config.py @@ -566,9 +566,9 @@ def print_previous_slurm_summary(self): formats = [] for k in self.info_keys: j = info_width[k] - k = '#N' if k == 'NNodes' else k + kh = '#N' if k == 'NNodes' else k formats.append(f"{{{k}:>{j}.{j}}}") - headers.append(f"{k:>{j}.{j}}") + headers.append(f"{kh:>{j}.{j}}") hlines.append("-" * j) table_header = '\n'.join((' '.join(headers), ' '.join(hlines))) From ce9369bf588e2a6c3a29f9064bbd41a0d9facaf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 13:28:45 +0100 Subject: [PATCH 201/245] Simplify job names --- cases/cosmo-ghg-spinup-test/cosmo_runjob.cfg | 2 +- cases/cosmo-ghg-spinup-test/int2lm_runjob.cfg | 2 +- cases/cosmo-ghg-test/cosmo_runjob.cfg | 2 +- cases/cosmo-ghg-test/int2lm_runjob.cfg | 2 +- cases/icon-art-global-test/icon_runjob.cfg | 2 +- cases/icon-art-oem-test/icon_runjob.cfg | 2 +- cases/icon-art-oem-test/icontools_remap_00_lbc_runjob.cfg | 2 +- cases/icon-art-oem-test/icontools_remap_ic_chem_runjob.cfg | 2 +- cases/icon-art-oem-test/icontools_remap_ic_runjob.cfg | 2 +- cases/icon-art-oem-test/icontools_remap_lbc_chem_runjob.cfg | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cases/cosmo-ghg-spinup-test/cosmo_runjob.cfg b/cases/cosmo-ghg-spinup-test/cosmo_runjob.cfg index ca14d636..28c0cbb6 100644 --- a/cases/cosmo-ghg-spinup-test/cosmo_runjob.cfg +++ b/cases/cosmo-ghg-spinup-test/cosmo_runjob.cfg @@ -1,5 +1,5 @@ #!/bin/bash -l -#SBATCH --job-name="cosmo_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" +#SBATCH --job-name=cosmo #SBATCH --account={cfg.compute_account} #SBATCH --time={walltime} #SBATCH --nodes={np_tot} diff --git a/cases/cosmo-ghg-spinup-test/int2lm_runjob.cfg b/cases/cosmo-ghg-spinup-test/int2lm_runjob.cfg index c3c80e53..9a3ae2e0 100644 --- a/cases/cosmo-ghg-spinup-test/int2lm_runjob.cfg +++ b/cases/cosmo-ghg-spinup-test/int2lm_runjob.cfg @@ -1,5 +1,5 @@ #!/bin/bash -l -#SBATCH --job-name=int2lm_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh} +#SBATCH --job-name=int2lm #SBATCH --account={cfg.compute_account} #SBATCH --time={walltime} #SBATCH --nodes={nodes} diff --git a/cases/cosmo-ghg-test/cosmo_runjob.cfg b/cases/cosmo-ghg-test/cosmo_runjob.cfg index ca14d636..28c0cbb6 100644 --- a/cases/cosmo-ghg-test/cosmo_runjob.cfg +++ b/cases/cosmo-ghg-test/cosmo_runjob.cfg @@ -1,5 +1,5 @@ #!/bin/bash -l -#SBATCH --job-name="cosmo_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" +#SBATCH --job-name=cosmo #SBATCH --account={cfg.compute_account} #SBATCH --time={walltime} #SBATCH --nodes={np_tot} diff --git a/cases/cosmo-ghg-test/int2lm_runjob.cfg b/cases/cosmo-ghg-test/int2lm_runjob.cfg index c3c80e53..9a3ae2e0 100644 --- a/cases/cosmo-ghg-test/int2lm_runjob.cfg +++ b/cases/cosmo-ghg-test/int2lm_runjob.cfg @@ -1,5 +1,5 @@ #!/bin/bash -l -#SBATCH --job-name=int2lm_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh} +#SBATCH --job-name=int2lm #SBATCH --account={cfg.compute_account} #SBATCH --time={walltime} #SBATCH --nodes={nodes} diff --git a/cases/icon-art-global-test/icon_runjob.cfg b/cases/icon-art-global-test/icon_runjob.cfg index e5334c64..99b0de5e 100644 --- a/cases/icon-art-global-test/icon_runjob.cfg +++ b/cases/icon-art-global-test/icon_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" +#SBATCH --job-name=icon #SBATCH --account={cfg.compute_account} #SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} diff --git a/cases/icon-art-oem-test/icon_runjob.cfg b/cases/icon-art-oem-test/icon_runjob.cfg index d64de85c..eec72b77 100644 --- a/cases/icon-art-oem-test/icon_runjob.cfg +++ b/cases/icon-art-oem-test/icon_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" +#SBATCH --job-name=icon #SBATCH --account={cfg.compute_account} #SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} diff --git a/cases/icon-art-oem-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-art-oem-test/icontools_remap_00_lbc_runjob.cfg index a93ac550..1f454071 100644 --- a/cases/icon-art-oem-test/icontools_remap_00_lbc_runjob.cfg +++ b/cases/icon-art-oem-test/icontools_remap_00_lbc_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconsub_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconsub #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-art-oem-test/icontools_remap_ic_chem_runjob.cfg b/cases/icon-art-oem-test/icontools_remap_ic_chem_runjob.cfg index 46ed8187..48fb9f3a 100644 --- a/cases/icon-art-oem-test/icontools_remap_ic_chem_runjob.cfg +++ b/cases/icon-art-oem-test/icontools_remap_ic_chem_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" +#SBATCH --job-name=iconremap_ic_chem #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-art-oem-test/icontools_remap_ic_runjob.cfg b/cases/icon-art-oem-test/icontools_remap_ic_runjob.cfg index e31b9eb5..e704afb8 100644 --- a/cases/icon-art-oem-test/icontools_remap_ic_runjob.cfg +++ b/cases/icon-art-oem-test/icontools_remap_ic_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconremap_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconremap_ic #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-art-oem-test/icontools_remap_lbc_chem_runjob.cfg b/cases/icon-art-oem-test/icontools_remap_lbc_chem_runjob.cfg index 2d21fe35..e629494b 100644 --- a/cases/icon-art-oem-test/icontools_remap_lbc_chem_runjob.cfg +++ b/cases/icon-art-oem-test/icontools_remap_lbc_chem_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.forecasttime}" +#SBATCH --job-name=iconremap_lbc #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} From 86b5f6859ac6cf3a0a44078b0009061fa1ef57d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Mon, 29 Jan 2024 13:34:59 +0100 Subject: [PATCH 202/245] Change missing job names --- cases/icon-art-oem-test/icontools_remap_lbc_rest_runjob.cfg | 2 +- cases/icon-seq-test/icon_runjob.cfg | 2 +- cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg | 2 +- cases/icon-seq-test/icontools_remap_ic_runjob.cfg | 2 +- cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg | 2 +- cases/icon-test/icon_runjob.cfg | 2 +- cases/icon-test/icontools_remap_00_lbc_runjob.cfg | 2 +- cases/icon-test/icontools_remap_ic_runjob.cfg | 2 +- cases/icon-test/icontools_remap_lbc_rest_runjob.cfg | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cases/icon-art-oem-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-art-oem-test/icontools_remap_lbc_rest_runjob.cfg index b0950efa..38dc6b07 100644 --- a/cases/icon-art-oem-test/icontools_remap_lbc_rest_runjob.cfg +++ b/cases/icon-art-oem-test/icontools_remap_lbc_rest_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconremap_lbc_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconremap_lbc #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-seq-test/icon_runjob.cfg b/cases/icon-seq-test/icon_runjob.cfg index 9fd03776..07fabaaf 100755 --- a/cases/icon-seq-test/icon_runjob.cfg +++ b/cases/icon-seq-test/icon_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}" +#SBATCH --job-name=icon #SBATCH --account={cfg.compute_account} #SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} diff --git a/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg index a93ac550..1f454071 100755 --- a/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg +++ b/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconsub_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconsub #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-seq-test/icontools_remap_ic_runjob.cfg b/cases/icon-seq-test/icontools_remap_ic_runjob.cfg index e31b9eb5..e704afb8 100755 --- a/cases/icon-seq-test/icontools_remap_ic_runjob.cfg +++ b/cases/icon-seq-test/icontools_remap_ic_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconremap_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconremap_ic #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg index b0950efa..38dc6b07 100755 --- a/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg +++ b/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconremap_lbc_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconremap_lbc #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-test/icon_runjob.cfg b/cases/icon-test/icon_runjob.cfg index 9fd03776..07fabaaf 100755 --- a/cases/icon-test/icon_runjob.cfg +++ b/cases/icon-test/icon_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="{cfg.casename}_{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}" +#SBATCH --job-name=icon #SBATCH --account={cfg.compute_account} #SBATCH --time={cfg.walltime_icon} #SBATCH --nodes={cfg.icon_np_tot} diff --git a/cases/icon-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-test/icontools_remap_00_lbc_runjob.cfg index a93ac550..1f454071 100755 --- a/cases/icon-test/icontools_remap_00_lbc_runjob.cfg +++ b/cases/icon-test/icontools_remap_00_lbc_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconsub_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconsub #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-test/icontools_remap_ic_runjob.cfg b/cases/icon-test/icontools_remap_ic_runjob.cfg index e31b9eb5..e704afb8 100755 --- a/cases/icon-test/icontools_remap_ic_runjob.cfg +++ b/cases/icon-test/icontools_remap_ic_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconremap_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconremap_ic #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} diff --git a/cases/icon-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-test/icontools_remap_lbc_rest_runjob.cfg index b0950efa..38dc6b07 100755 --- a/cases/icon-test/icontools_remap_lbc_rest_runjob.cfg +++ b/cases/icon-test/icontools_remap_lbc_rest_runjob.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env bash -#SBATCH --job-name="iconremap_lbc_{cfg.startdate_sim_yyyymmddhh}" +#SBATCH --job-name=iconremap_lbc #SBATCH --account={cfg.compute_account} #SBATCH --chdir={cfg.icon_work} #SBATCH --partition={cfg.compute_queue} From 97b2fa5ca2cc3617004b287ca04d70d043585455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 09:24:30 +0100 Subject: [PATCH 203/245] Fix syntax error from commit 8810dcf --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 565f882b..37c4b9aa 100644 --- a/config.py +++ b/config.py @@ -379,7 +379,7 @@ def get_dep_cmd(self, job_name, add_dep=None): # sequential case return '--wait' - def submit(self, job_name, script, add_dep=None, logfile=None): + def submit(self, job_name, script, add_dep=None): """Submit job with dependencies""" script_path = Path(script) sbatch_cmd = ['sbatch', '--parsable'] @@ -391,7 +391,7 @@ def submit(self, job_name, script, add_dep=None, logfile=None): result = run(sbatch_cmd, cwd=script_path.parent, capture_output=True, check=True) except CalledProcessError as e: - with open self.logfile('a') as f: + with open(self.logfile('a')) as f: f.write(e) raise(e) From 4b01685af3f57ccc88f8ea7659692e40ca728383 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 08:24:51 +0000 Subject: [PATCH 204/245] GitHub Action: Apply Pep8-formatting --- config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 37c4b9aa..bab601d7 100644 --- a/config.py +++ b/config.py @@ -388,12 +388,14 @@ def submit(self, job_name, script, add_dep=None): sbatch_cmd.append(script_path.name) try: - result = run(sbatch_cmd, cwd=script_path.parent, - capture_output=True, check=True) + result = run(sbatch_cmd, + cwd=script_path.parent, + capture_output=True, + check=True) except CalledProcessError as e: with open(self.logfile('a')) as f: f.write(e) - raise(e) + raise (e) job_id = int(result.stdout) print(f' └── Submitted batch job {job_id}') From 3a65efed0d6f9874ba4a0ef9869b6a7f37ba1ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 09:57:15 +0100 Subject: [PATCH 205/245] Just jobname for jobs --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index bab601d7..71e25618 100644 --- a/config.py +++ b/config.py @@ -414,7 +414,7 @@ def create_sbatch_script(self, job_name): walltime = getattr(self, 'walltime', {}).get(job_name, "00:30:00") script_lines = [ '#!/usr/bin/env bash', - f'#SBATCH --job-name="{job_name}_{self.chunk_id}"', + f'#SBATCH --job-name={job_name}', '#SBATCH --nodes=1', f'#SBATCH --time={walltime}', f'#SBATCH --output={self.logfile}', From 90793ab102f8c9cb570689beb1eb27339e755017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 10:12:27 +0100 Subject: [PATCH 206/245] Cleaner console output --- run_chain.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/run_chain.py b/run_chain.py index 2e465fa4..dc41670d 100755 --- a/run_chain.py +++ b/run_chain.py @@ -10,7 +10,6 @@ import time import shutil import argparse -import inspect import jobs from jobs import tools @@ -231,11 +230,11 @@ def run_chunk(cfg, force, resume): for job_name in cfg.jobs: if (cfg.log_finished_dir / job_name).exists() and not force: # Skip job if already finished - print(f' └── Skip "{job_name}" for chunk "{cfg.chunk_id}"') + print(f' └── Skipping "{job_name}" job') skip = True else: print( - f' └── Submit "{job_name}" for chunk "{cfg.chunk_id}"') + f' └── Starting "{job_name}" job') # Logfile settings cfg.logfile = cfg.log_working_dir / job_name @@ -370,7 +369,7 @@ def restart_runs(cfg, force, resume): # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." - print(f"└── Starting chunk with startdate {cfg.startdate_sim}") + print(f'└── Starting chunk "{cfg.chunk_id}"') run_chunk(cfg=cfg, force=force, resume=resume) From f8551b9ad8b01523370b2a8870498779490bae40 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 09:12:49 +0000 Subject: [PATCH 207/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index dc41670d..3f357e6d 100755 --- a/run_chain.py +++ b/run_chain.py @@ -233,8 +233,7 @@ def run_chunk(cfg, force, resume): print(f' └── Skipping "{job_name}" job') skip = True else: - print( - f' └── Starting "{job_name}" job') + print(f' └── Starting "{job_name}" job') # Logfile settings cfg.logfile = cfg.log_working_dir / job_name From 06729b46921a26e37441c832afc0a2a3946526f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 10:47:58 +0100 Subject: [PATCH 208/245] Unify restart and spinup runs --- run_chain.py | 94 +++++++++++++++------------------------------------- 1 file changed, 26 insertions(+), 68 deletions(-) diff --git a/run_chain.py b/run_chain.py index 3f357e6d..d67854c2 100755 --- a/run_chain.py +++ b/run_chain.py @@ -347,13 +347,32 @@ def restart_runs(cfg, force, resume): if not cfg.chunks: for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, cfg.restart_step_hours): - enddate_sim = startdate_sim + timedelta( - hours=cfg.restart_step_hours) + if 'spinup' in cfg.workflow['features'] and hasattr(cfg, 'spinup'): + if startdate_sim == cfg.startdate: + cfg.first_one = True + cfg.second_one = False + cfg.lrestart = '.FALSE.' + elif startdate_sim == cfg.startdate + timedelta( + hours=cfg.restart_step_hours): + cfg.first_one = False + cfg.second_one = True + cfg.lrestart = '.TRUE.' + startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) + else: + cfg.first_one = False + cfg.second_one = False + cfg.lrestart = '.TRUE.' + startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) + else: + enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) + startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") chunk_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" + if enddate_sim > cfg.enddate: continue + cfg.chunks.append(chunk_id) for chunk_id in cfg.chunks: @@ -365,70 +384,15 @@ def restart_runs(cfg, force, resume): cfg.enddate_sim = datetime.strptime( cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) - # Set restart variable (only takes effect for ICON) - cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." + if 'spinup' not in cfg.workflow['features'] and not hasattr(cfg, 'spinup'): + # Set restart variable (only takes effect for ICON) + cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." print(f'└── Starting chunk "{cfg.chunk_id}"') run_chunk(cfg=cfg, force=force, resume=resume) -def restart_runs_spinup(cfg, force, resume): - """Start subchains in specified intervals and manage restarts with spin-up. - - This function slices the total runtime of the processing chain according to the - `cfg.restart_step_hours` configuration. It calls `run_chunk()` for each - specified interval, managing restarts with spin-up. - - Parameters - ---------- - cfg : Config - Object holding all user-configuration parameters as attributes. - force : bool - If True, it will force the execution of jobs regardless of their completion status. - resume : bool - If True, it will resume the last unfinished job. - - Notes - ----- - - The function iterates over specified intervals, calling `run_chunk()` for each. - - It manages restart settings and logging for each subchain, including spin-up. - """ - for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, - cfg.restart_step_hours): - if startdate_sim == cfg.startdate: - cfg.first_one = True - cfg.second_one = False - cfg.lrestart = '.FALSE.' - run_time = cfg.restart_step_hours - startdate_sim_spinup = startdate_sim - elif startdate_sim == cfg.startdate + timedelta( - hours=cfg.restart_step_hours): - cfg.first_one = False - cfg.second_one = True - cfg.lrestart = '.TRUE.' - run_time = cfg.restart_step_hours + cfg.spinup - startdate_sim_spinup = startdate_sim - timedelta(hours=cfg.spinup) - else: - cfg.first_one = False - cfg.second_one = False - cfg.lrestart = '.TRUE.' - run_time = cfg.restart_step_hours + cfg.spinup - startdate_sim_spinup = startdate_sim - timedelta(hours=cfg.spinup) - - # If current enddate is later than global enddate, skip - enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) - if enddate_sim > cfg.enddate: - continue - - print(f'Runtime of sub-simulation: {run_time} h') - - cfg.startdate_sim = startdate_sim_spinup - cfg.enddate_sim = enddate_sim - - run_chunk(cfg=cfg, force=force, resume=resume) - - def main(): """Main script for running a processing chain. @@ -499,14 +463,8 @@ def main(): # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: - if hasattr(cfg, 'spinup'): - print("Using spin-up restarts.") - restart_runs_spinup(cfg=cfg, - force=args.force, - resume=args.resume) - else: - print("Using built-in model restarts.") - restart_runs(cfg=cfg, force=args.force, resume=args.resume) + print("Using built-in model restarts.") + restart_runs(cfg=cfg, force=args.force, resume=args.resume) else: print("No restarts are used.") cfg.startdate_sim = cfg.startdate From 2d3fb0b16b1567142142838dd135370f388d76ca Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 09:48:23 +0000 Subject: [PATCH 209/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index d67854c2..68a955a7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -364,7 +364,8 @@ def restart_runs(cfg, force, resume): cfg.lrestart = '.TRUE.' startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) else: - enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) + enddate_sim = startdate_sim + timedelta( + hours=cfg.restart_step_hours) startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") @@ -384,7 +385,8 @@ def restart_runs(cfg, force, resume): cfg.enddate_sim = datetime.strptime( cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) - if 'spinup' not in cfg.workflow['features'] and not hasattr(cfg, 'spinup'): + if 'spinup' not in cfg.workflow['features'] and not hasattr( + cfg, 'spinup'): # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." From 746300981d2bf68a775be003939db9f4f5c70a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 10:51:22 +0100 Subject: [PATCH 210/245] Complete config file for spinup test --- cases/cosmo-ghg-spinup-test/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cases/cosmo-ghg-spinup-test/config.yaml b/cases/cosmo-ghg-spinup-test/config.yaml index 011e0f02..271c17a9 100644 --- a/cases/cosmo-ghg-spinup-test/config.yaml +++ b/cases/cosmo-ghg-spinup-test/config.yaml @@ -2,6 +2,8 @@ workflow: cosmo-ghg constraint: gpu +run_on: gpu +compute_queue: normal ntasks_per_node: 12 restart_step: PT6H spinup: 3 From 44e3c7480938bc1e8b46b416a388c6b6c29f9025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 11:13:14 +0100 Subject: [PATCH 211/245] Fix for spinup --- run_chain.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run_chain.py b/run_chain.py index 68a955a7..874718fa 100755 --- a/run_chain.py +++ b/run_chain.py @@ -363,10 +363,8 @@ def restart_runs(cfg, force, resume): cfg.second_one = False cfg.lrestart = '.TRUE.' startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) - else: - enddate_sim = startdate_sim + timedelta( - hours=cfg.restart_step_hours) + enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") chunk_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" From 79d5f94ae7e0537d882ee0ba143d1e738fc7ca04 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 10:13:40 +0000 Subject: [PATCH 212/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 874718fa..869a9bb7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -364,7 +364,8 @@ def restart_runs(cfg, force, resume): cfg.lrestart = '.TRUE.' startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) - enddate_sim = startdate_sim + timedelta(hours=cfg.restart_step_hours) + enddate_sim = startdate_sim + timedelta( + hours=cfg.restart_step_hours) startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") chunk_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" From 287ff6b5b86dfe409c946acd25381b9224197df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 11:28:24 +0100 Subject: [PATCH 213/245] Fix 2 for spinup --- run_chain.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/run_chain.py b/run_chain.py index 869a9bb7..90b10e71 100755 --- a/run_chain.py +++ b/run_chain.py @@ -348,20 +348,7 @@ def restart_runs(cfg, force, resume): for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, cfg.restart_step_hours): if 'spinup' in cfg.workflow['features'] and hasattr(cfg, 'spinup'): - if startdate_sim == cfg.startdate: - cfg.first_one = True - cfg.second_one = False - cfg.lrestart = '.FALSE.' - elif startdate_sim == cfg.startdate + timedelta( - hours=cfg.restart_step_hours): - cfg.first_one = False - cfg.second_one = True - cfg.lrestart = '.TRUE.' - startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) - else: - cfg.first_one = False - cfg.second_one = False - cfg.lrestart = '.TRUE.' + if startdate_sim > cfg.startdate: startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) enddate_sim = startdate_sim + timedelta( @@ -384,8 +371,23 @@ def restart_runs(cfg, force, resume): cfg.enddate_sim = datetime.strptime( cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) - if 'spinup' not in cfg.workflow['features'] and not hasattr( - cfg, 'spinup'): + if 'spinup' in cfg.workflow['features'] and hasattr(cfg, 'spinup'): + print("Using spin-up restarts.") + if cfg.startdate_sim == cfg.startdate: + cfg.first_one = True + cfg.second_one = False + cfg.lrestart = '.FALSE.' + elif cfg.startdate_sim == cfg.startdate + timedelta( + hours=cfg.restart_step_hours): + cfg.first_one = False + cfg.second_one = True + cfg.lrestart = '.TRUE.' + else: + cfg.first_one = False + cfg.second_one = False + cfg.lrestart = '.TRUE.' + else: + print("Using built-in model restarts.") # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." @@ -464,7 +466,6 @@ def main(): # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: - print("Using built-in model restarts.") restart_runs(cfg=cfg, force=args.force, resume=args.resume) else: print("No restarts are used.") From 365cad8babc43db9f56a9f199db9575906aedc20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 11:44:33 +0100 Subject: [PATCH 214/245] Fancy formatting --- run_chain.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/run_chain.py b/run_chain.py index 90b10e71..9daff777 100755 --- a/run_chain.py +++ b/run_chain.py @@ -455,14 +455,13 @@ def main(): cfg.print_config() tools.create_dir(cfg.case_root, "case_root") - print( - f"Starting chain for case {casename} and workflow {cfg.workflow_name}" - ) - if cfg.is_async: - print("Running the Processing Chain in asynchronous mode.") - else: - print("Running the Processing Chain in sequential mode.") + print("╔════════════════════════════════════════╗") + print("║ Starting Processing Chain ║") + print("║════════════════════════════════════════║") + print(f"║ Case: {casename: <34} ║") + print(f"║ Workflow: {cfg.workflow_name: <28} ║") + print("╚════════════════════════════════════════╝") # Check for restart compatibility and spinup if 'restart' in cfg.workflow['features']: @@ -473,7 +472,9 @@ def main(): cfg.enddate_sim = cfg.enddate run_chunk(cfg=cfg, force=args.force, resume=args.resume) - print('>>> Finished the processing chain successfully <<<') + print("╔════════════════════════════════════════╗") + print("║ Processing Chain Completed ║") + print("╚════════════════════════════════════════╝") if __name__ == '__main__': From c7f68266b568fa51a4ba8ee1d1883ddb772109e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 12:15:43 +0100 Subject: [PATCH 215/245] Define separate spinup workflow --- cases/cosmo-ghg-spinup-test/config.yaml | 2 +- workflows.yaml | 49 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/cases/cosmo-ghg-spinup-test/config.yaml b/cases/cosmo-ghg-spinup-test/config.yaml index 271c17a9..b52048ba 100644 --- a/cases/cosmo-ghg-spinup-test/config.yaml +++ b/cases/cosmo-ghg-spinup-test/config.yaml @@ -1,6 +1,6 @@ # Configuration file for the 'cosmo-ghg-spinup-test' case with COSMO-GHG -workflow: cosmo-ghg +workflow: cosmo-ghg-spinup constraint: gpu run_on: gpu compute_queue: normal diff --git a/workflows.yaml b/workflows.yaml index 8ec87076..c5c574c0 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -55,6 +55,55 @@ cosmo-ghg: current: - cosmo +cosmo-ghg-spinup: + features: + - restart + - tracers + - spinup + jobs: + - prepare_cosmo + - emissions + - biofluxes + - oem + - online_vprm + - int2lm + - post_int2lm + - cosmo + - post_cosmo + dependencies: + emissions: + current: + - prepare_cosmo + biofluxes: + current: + - prepare_cosmo + oem: + current: + - prepare_cosmo + online_vprm: + current: + - prepare_cosmo + int2lm: + current: + - prepare_cosmo + - emissions + - biofluxes + - oem + - online_vprm + post_int2lm: + current: + - int2lm + previous: + - cosmo + cosmo: + current: + - post_int2lm + previous: + - cosmo + post_cosmo: + current: + - cosmo + cosmo-art: features: - nesting From 5afb56eca97c634daedb6e36fc0ab73ea19f061e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 12:17:40 +0100 Subject: [PATCH 216/245] Fix formatting (hopefully) --- run_chain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index 9daff777..2f97020f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -459,8 +459,8 @@ def main(): print("╔════════════════════════════════════════╗") print("║ Starting Processing Chain ║") print("║════════════════════════════════════════║") - print(f"║ Case: {casename: <34} ║") - print(f"║ Workflow: {cfg.workflow_name: <28} ║") + print(f"║ Case: {casename: <31} ║") + print(f"║ Workflow: {cfg.workflow_name: <27} ║") print("╚════════════════════════════════════════╝") # Check for restart compatibility and spinup From 0cf4a039c8747aafb478dd10d9919263816e7352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 12:23:53 +0100 Subject: [PATCH 217/245] Aligning case + workflow --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 2f97020f..eafc35be 100755 --- a/run_chain.py +++ b/run_chain.py @@ -459,7 +459,7 @@ def main(): print("╔════════════════════════════════════════╗") print("║ Starting Processing Chain ║") print("║════════════════════════════════════════║") - print(f"║ Case: {casename: <31} ║") + print(f"║ Case: {casename: <27} ║") print(f"║ Workflow: {cfg.workflow_name: <27} ║") print("╚════════════════════════════════════════╝") From b1448d3956506498d74e6ddc716e0188256540ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 12:36:42 +0100 Subject: [PATCH 218/245] Custom workflow_name and improve check --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 71e25618..315ce3c8 100644 --- a/config.py +++ b/config.py @@ -157,7 +157,7 @@ def set_node_info(self): If the 'constraint' or 'run_on' configuration values are invalid. """ if self.constraint == 'gpu': - if self.workflow_name.startswith('icon'): + if hasattr(self, 'icon'): if self.run_on == 'gpu': self.ntasks_per_node = 1 elif self.run_on == 'cpu': @@ -189,7 +189,7 @@ def set_workflow(self): self.workflow = workflows[self.workflow_name] # Otherwise, use custom workflow from config.yaml directly elif isinstance(self.workflow, dict): - self.workflow_name = self.casename + self.workflow_name = 'custom' else: raise InvalidWorkflowType( "Invalid workflow type. Must be either a string or a dictionary." From 2831e9878dd7bf12ac30eebfdaac9047f3c8a336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 13:26:48 +0100 Subject: [PATCH 219/245] Remove workflow_name check in jobs --- jobs/cosmo.py | 12 ++++++------ jobs/emissions.py | 2 +- jobs/icontools.py | 2 +- jobs/int2lm.py | 2 +- jobs/oem.py | 4 ++-- jobs/prepare_cosmo.py | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 3ebfa341..d987e2e2 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -128,18 +128,16 @@ def main(cfg): tools.create_dir(cfg.cosmo_restart_out, "cosmo_restart_out") # Copy cosmo executable - cfg.cosmo['execname'] = cfg.workflow_name.lower() + cfg.cosmo['execname'] = 'cosmo.exe' tools.copy_file(cfg.cosmo['binary_file'], os.path.join(cfg.cosmo_run, cfg.cosmo['execname'])) # Prepare namelist and submit job tracer_csvfile = os.path.join(cfg.chain_src_dir, 'cases', cfg.casename, 'cosmo_tracers.csv') - if cfg.workflow_name == 'cosmo': - namelist_names = ['ORG', 'IO', 'DYN', 'PHY', 'DIA', 'ASS', 'SAT'] - elif cfg.workflow_name == 'cosmo-ghg': + if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): namelist_names = ['AF', 'ORG', 'IO', 'DYN', 'GHG', 'PHY', 'DIA', 'ASS'] - elif cfg.workflow_name == 'cosmo-art': + if hasattr(cfg, 'photo_rate'): namelist_names = [ 'ART', 'ASS', 'DIA', 'DYN', 'EPS', 'INI', 'IO', 'ORG', 'PHY' ] @@ -147,6 +145,8 @@ def main(cfg): # When doing online emissions in COSMO-ART, an additional # namelist is required namelist_names += ['OAE'] + elif hasattr(cfg, 'cosmo'): + namelist_names = ['ORG', 'IO', 'DYN', 'PHY', 'DIA', 'ASS', 'SAT'] for section in namelist_names: namelist_file = os.path.join( @@ -178,7 +178,7 @@ def main(cfg): # Append INPUT_GHG namelist with tracer definitions from csv file if os.path.isfile(tracer_csvfile): - if cfg.workflow_name == 'cosmo-ghg': + if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): input_ghg_filename = os.path.join(cfg.cosmo_run, 'INPUT_GHG') write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg) diff --git a/jobs/emissions.py b/jobs/emissions.py index 0b252c4c..27516da3 100644 --- a/jobs/emissions.py +++ b/jobs/emissions.py @@ -68,5 +68,5 @@ def main(cfg): # convert grid_mapping_name from string (NF90_STRING) to char # (NF90_CHAR) (needed for int2lm to work) - if cfg.workflow_name.startswith('cosmo'): + if hasattr(cfg, 'cosmo'): tools.string2char.main(dest_path) diff --git a/jobs/icontools.py b/jobs/icontools.py index c68dc485..2ee7c2eb 100644 --- a/jobs/icontools.py +++ b/jobs/icontools.py @@ -32,7 +32,7 @@ def main(cfg): cfg.meteo['inc']): meteo_file = cfg.icon_input_icbc / ( cfg.meteo['prefix'] + time.strftime(cfg.meteo['nameformat'])) - if cfg.workflow_name == 'icon-art' or cfg.workflow_name == 'icon-art-oem': + if hasattr(cfg, 'art_input_folder'): chem_file = cfg.icon_input_icbc / ( cfg.chem['prefix'] + time.strftime(cfg.chem_nameformat)) datafile_list_chem.append(str(chem_file) + cfg.chem['suffix']) diff --git a/jobs/int2lm.py b/jobs/int2lm.py index c79169d8..83249fa3 100644 --- a/jobs/int2lm.py +++ b/jobs/int2lm.py @@ -68,7 +68,7 @@ def main(cfg): extpar_dir) # Copy landuse and plant-functional-type files - if cfg.workflow_name == 'cosmo-art': + if hasattr(cfg, 'photo_rate'): lu_file_src = cfg.int2lm['lu_file'] lu_file_dst = os.path.join(extpar_dir, 'landuse.nc') tools.copy_file(lu_file_src, lu_file_dst) diff --git a/jobs/oem.py b/jobs/oem.py index 8ad1511e..e9cf42d7 100644 --- a/jobs/oem.py +++ b/jobs/oem.py @@ -52,9 +52,9 @@ def main(cfg): raise RuntimeError("At least one of (hod/dow/moy) or (hoy) netcdfs " " have to be given for online emissions") - if cfg.workflow_name.startswith('icon'): + if hasattr(cfg, 'icon'): input_dir = cfg.icon_input - else: + elif hasattr(cfg, 'cosmo'): input_dir = cfg.cosmo_input dest_dir = os.path.join(input_dir, "oem") tools.create_dir(dest_dir, "online emissions input") diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 444d78e8..8e3ecb7f 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -161,7 +161,7 @@ def main(cfg): # Other IC/BC data inv_to_process = [] - if cfg.workflow_name == 'cosmo-ghg': + if hasattr(cfg, 'cams'): try: CAMS = dict(fullname="CAMS", nickname="cams", @@ -185,7 +185,7 @@ def main(cfg): inv_to_process.append(CT) except AttributeError: pass - elif cfg.workflow_name == 'cosmo-art': + elif hasattr(cfg, 'mozart'): try: MOZART = dict(fullname='MOZART', nickname='mozart', @@ -200,7 +200,7 @@ def main(cfg): except AttributeError: pass - if cfg.workflow_name == 'cosmo-ghg' or cfg.workflow_name == 'cosmo-art': + if inv_to_process: logging.info("Processing " + ", ".join([i["fullname"] for i in inv_to_process]) + " data") From 126b9c1a0a105b7cb80c97ba5f1ef7ddb054f643 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 12:27:13 +0000 Subject: [PATCH 220/245] GitHub Action: Apply Pep8-formatting --- jobs/cosmo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index d987e2e2..53957fd3 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -178,7 +178,7 @@ def main(cfg): # Append INPUT_GHG namelist with tracer definitions from csv file if os.path.isfile(tracer_csvfile): - if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): + if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): input_ghg_filename = os.path.join(cfg.cosmo_run, 'INPUT_GHG') write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg) From 74cd9f810869ce6038e77d85ea695ee209dff0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 13:53:56 +0100 Subject: [PATCH 221/245] Bugfix in cosmo job --- jobs/cosmo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 53957fd3..1afdca96 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -137,7 +137,7 @@ def main(cfg): 'cosmo_tracers.csv') if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): namelist_names = ['AF', 'ORG', 'IO', 'DYN', 'GHG', 'PHY', 'DIA', 'ASS'] - if hasattr(cfg, 'photo_rate'): + elif hasattr(cfg, 'photo_rate'): namelist_names = [ 'ART', 'ASS', 'DIA', 'DYN', 'EPS', 'INI', 'IO', 'ORG', 'PHY' ] From 0cd732e6c3cc850455ff98895b91efc85c7e7a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 13:55:49 +0100 Subject: [PATCH 222/245] Remove restart info --- run_chain.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index eafc35be..c31b16e7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -372,7 +372,6 @@ def restart_runs(cfg, force, resume): cfg.enddate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) if 'spinup' in cfg.workflow['features'] and hasattr(cfg, 'spinup'): - print("Using spin-up restarts.") if cfg.startdate_sim == cfg.startdate: cfg.first_one = True cfg.second_one = False @@ -387,7 +386,6 @@ def restart_runs(cfg, force, resume): cfg.second_one = False cfg.lrestart = '.TRUE.' else: - print("Using built-in model restarts.") # Set restart variable (only takes effect for ICON) cfg.lrestart = ".FALSE." if cfg.startdate_sim == cfg.startdate else ".TRUE." From a676d752c980607a13feaac1d2d253954ff40d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 14:36:30 +0100 Subject: [PATCH 223/245] Compute chunks separately --- config.py | 21 +++++++++++++++++++++ run_chain.py | 21 ++------------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/config.py b/config.py index 315ce3c8..0057bec3 100644 --- a/config.py +++ b/config.py @@ -1,6 +1,8 @@ from subprocess import run, CalledProcessError import os import yaml +from datetime import timedelta + from jobs import tools from pathlib import Path @@ -336,6 +338,25 @@ def format_duration(self, duration): formatted_duration = f"{int(days)}d {int(hours)}h {int(minutes)}m {int(seconds)}s" return formatted_duration + def get_chunks(self): + self.chunks = [] + for startdate_sim in tools.iter_hours(self.startdate, self.enddate, + self.restart_step_hours): + if 'spinup' in self.workflow['features'] and hasattr(self, 'spinup'): + if startdate_sim > self.startdate: + startdate_sim = startdate_sim - timedelta(hours=self.spinup) + + enddate_sim = startdate_sim + timedelta( + hours=self.restart_step_hours) + startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") + enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") + chunk_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" + + if enddate_sim > self.enddate: + continue + + self.chunks.append(chunk_id) + def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" # Initial list of dependencies diff --git a/run_chain.py b/run_chain.py index c31b16e7..82acfef3 100755 --- a/run_chain.py +++ b/run_chain.py @@ -344,23 +344,6 @@ def restart_runs(cfg, force, resume): - The function iterates over specified intervals, calling `run_chunk()` for each. - It manages restart settings and logging for each subchain. """ - if not cfg.chunks: - for startdate_sim in tools.iter_hours(cfg.startdate, cfg.enddate, - cfg.restart_step_hours): - if 'spinup' in cfg.workflow['features'] and hasattr(cfg, 'spinup'): - if startdate_sim > cfg.startdate: - startdate_sim = startdate_sim - timedelta(hours=cfg.spinup) - - enddate_sim = startdate_sim + timedelta( - hours=cfg.restart_step_hours) - startdate_sim_yyyymmddhh = startdate_sim.strftime("%Y%m%d%H") - enddate_sim_yyyymmddhh = enddate_sim.strftime("%Y%m%d%H") - chunk_id = f"{startdate_sim_yyyymmddhh}_{enddate_sim_yyyymmddhh}" - - if enddate_sim > cfg.enddate: - continue - - cfg.chunks.append(chunk_id) for chunk_id in cfg.chunks: cfg.chunk_id = chunk_id @@ -439,9 +422,9 @@ def main(): else: cfg.jobs = args.job_list - # Check if chunks are set or if all are used + # Check if chunks are set or if all are used and compute them if args.chunk_list is None: - cfg.chunks = [] + cfg.get_chunks() else: cfg.chunks = args.chunk_list From 99e5cba2e265358c58c1300a1a579c4f555e32af Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 13:36:52 +0000 Subject: [PATCH 224/245] GitHub Action: Apply Pep8-formatting --- config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 0057bec3..81e91f4d 100644 --- a/config.py +++ b/config.py @@ -3,7 +3,6 @@ import yaml from datetime import timedelta - from jobs import tools from pathlib import Path @@ -342,9 +341,11 @@ def get_chunks(self): self.chunks = [] for startdate_sim in tools.iter_hours(self.startdate, self.enddate, self.restart_step_hours): - if 'spinup' in self.workflow['features'] and hasattr(self, 'spinup'): + if 'spinup' in self.workflow['features'] and hasattr( + self, 'spinup'): if startdate_sim > self.startdate: - startdate_sim = startdate_sim - timedelta(hours=self.spinup) + startdate_sim = startdate_sim - timedelta( + hours=self.spinup) enddate_sim = startdate_sim + timedelta( hours=self.restart_step_hours) From f1d3caf2cca4ee5ee4510f078338bb4c6ce4f368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 14:47:37 +0100 Subject: [PATCH 225/245] Add function to get previous chunk ID --- config.py | 10 ++++++++++ jobs/cosmo.py | 2 +- run_chain.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index 0057bec3..145e7920 100644 --- a/config.py +++ b/config.py @@ -357,6 +357,16 @@ def get_chunks(self): self.chunks.append(chunk_id) + def previous_chunk_id(self, current_chunk_id): + """Get the previous chunk ID based on the current chunk ID.""" + try: + index = self.chunks.index(current_chunk_id) + if index > 0: + return self.chunks[index - 1] + except ValueError: + # If the current_chunk_id is not in the list or it's the first chunk + return None + def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" # Initial list of dependencies diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 1afdca96..83dad5aa 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -178,7 +178,7 @@ def main(cfg): # Append INPUT_GHG namelist with tracer definitions from csv file if os.path.isfile(tracer_csvfile): - if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): + if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): input_ghg_filename = os.path.join(cfg.cosmo_run, 'INPUT_GHG') write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg) diff --git a/run_chain.py b/run_chain.py index 82acfef3..9d4ca5e5 100755 --- a/run_chain.py +++ b/run_chain.py @@ -347,6 +347,7 @@ def restart_runs(cfg, force, resume): for chunk_id in cfg.chunks: cfg.chunk_id = chunk_id + cfg.chunk_id_prev = cfg.previous_chunk_id(cfg.chunk_id) cfg.startdate_sim_yyyymmddhh = chunk_id[0:10] cfg.enddate_sim_yyyymmddhh = chunk_id[-10:] cfg.startdate_sim = datetime.strptime( From 121c44f280f53efb51b852572b4f9de237b03ca1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 13:48:07 +0000 Subject: [PATCH 226/245] GitHub Action: Apply Pep8-formatting --- jobs/cosmo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/cosmo.py b/jobs/cosmo.py index 83dad5aa..1afdca96 100644 --- a/jobs/cosmo.py +++ b/jobs/cosmo.py @@ -178,7 +178,7 @@ def main(cfg): # Append INPUT_GHG namelist with tracer definitions from csv file if os.path.isfile(tracer_csvfile): - if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): + if hasattr(cfg, 'cams') or hasattr(cfg, 'mozart'): input_ghg_filename = os.path.join(cfg.cosmo_run, 'INPUT_GHG') write_cosmo_input_ghg.main(tracer_csvfile, input_ghg_filename, cfg) From 38e2c36c995b71adacc4c9c31d3ca04a26f91199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 14:53:23 +0100 Subject: [PATCH 227/245] Directly get previous chunk id --- config.py | 6 +++--- run_chain.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 37bf7347..e418828b 100644 --- a/config.py +++ b/config.py @@ -358,15 +358,15 @@ def get_chunks(self): self.chunks.append(chunk_id) - def previous_chunk_id(self, current_chunk_id): + def get_previous_chunk_id(self, current_chunk_id): """Get the previous chunk ID based on the current chunk ID.""" try: index = self.chunks.index(current_chunk_id) if index > 0: - return self.chunks[index - 1] + self.chunk_id_prev = self.chunks[index - 1] except ValueError: # If the current_chunk_id is not in the list or it's the first chunk - return None + self.chunk_id_prev = None def get_dep_ids(self, job_name, add_dep=None): """Get dependency job ids for `job_name`""" diff --git a/run_chain.py b/run_chain.py index 9d4ca5e5..97e700bf 100755 --- a/run_chain.py +++ b/run_chain.py @@ -347,7 +347,7 @@ def restart_runs(cfg, force, resume): for chunk_id in cfg.chunks: cfg.chunk_id = chunk_id - cfg.chunk_id_prev = cfg.previous_chunk_id(cfg.chunk_id) + cfg.get_previous_chunk_id(cfg.chunk_id) cfg.startdate_sim_yyyymmddhh = chunk_id[0:10] cfg.enddate_sim_yyyymmddhh = chunk_id[-10:] cfg.startdate_sim = datetime.strptime( From 59521fad17fe75b9007a71512af241c574b771b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 17:55:52 +0100 Subject: [PATCH 228/245] Further cleanup and refactoring --- jobs/prepare_cosmo.py | 10 +++++ jobs/reduce_output.py | 7 ++++ run_chain.py | 87 ++++++++----------------------------------- 3 files changed, 33 insertions(+), 71 deletions(-) diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index 8e3ecb7f..f0208333 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -4,6 +4,7 @@ from pathlib import Path import logging import csv +import os from datetime import timedelta from . import tools @@ -51,6 +52,15 @@ def set_cfg_variables(cfg): cfg.lasync_io = '.TRUE.' cfg.num_iope_percomm = 1 + # If nested run: use output of mother-simulation + if 'nesting' in cfg.workflow['features'] and not os.path.isdir( + cfg.meteo.dir): + # if ifs_hres_dir doesn't point to a directory, + # it is the name of the mother run + mother_name = cfg.meteo.dir + cfg.meteo.dir = cfg.work_root / mother_name / cfg.chunk_id / 'cosmo' / 'output' + cfg.meteo.inc = 1 + cfg.meteo.prefix = 'lffd' def main(cfg): """ diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index 7163b904..435dbfeb 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -69,6 +69,13 @@ def main(cfg): # Wait for Cosmo to finish first tools.check_job_completion(cfg.log_finished_dir, "cosmo") + + # Number of levels and switch for unit conversion for 'reduce_output' job + if not hasattr(cfg, 'output_levels'): + cfg.output_levels = -1 + if not hasattr(cfg, 'convert_gas'): + cfg.convert_gas = True + """Get list of constant files""" cfiles = [] read_cfile = False diff --git a/run_chain.py b/run_chain.py index 97e700bf..0f7e5a72 100755 --- a/run_chain.py +++ b/run_chain.py @@ -140,87 +140,27 @@ def run_chunk(cfg, force, resume): cfg.forecasttime = (cfg.enddate_sim - cfg.startdate_sim).total_seconds() / 3600 - # String variables for startdate_sim - cfg.startdate_sim_yyyymmddhh = cfg.startdate_sim.strftime('%Y%m%d%H') - cfg.enddate_sim_yyyymmddhh = cfg.enddate_sim.strftime('%Y%m%d%H') - - # Folder naming and structure - cfg.chunk_id = f'{cfg.startdate_sim_yyyymmddhh}_{cfg.enddate_sim_yyyymmddhh}' + # Logging cfg.chain_root = cfg.work_root / cfg.casename / cfg.chunk_id + cfg.log_working_dir = cfg.chain_root / 'checkpoints' / 'working' + cfg.log_finished_dir = cfg.chain_root / 'checkpoints' / 'finished' - # Config variables for spinup runs (datetimes, job-id, etc.) + # Create working directories + tools.create_dir(cfg.chain_root, "chain_root") + tools.create_dir(cfg.log_working_dir, "log_working") + tools.create_dir(cfg.log_finished_dir, "log_finished") + + # Config variables for spinup and restart runs if hasattr(cfg, 'spinup'): - if cfg.first_one: # first run in spinup - cfg.chain_root_prev = None - else: # consecutive runs in spinup - cfg.startdate_sim_yyyymmddhh = cfg.startdate_sim.strftime( - '%Y%m%d%H') - enddate_sim_yyyymmddhh_prev = ( - cfg.enddate_sim - - timedelta(hours=cfg.restart_step_hours)).strftime('%Y%m%d%H') - - if cfg.second_one: - startdate_sim_yyyymmddhh_prev = (cfg.enddate_sim - timedelta( - hours=2 * cfg.restart_step_hours)).strftime('%Y%m%d%H') - else: # all other runs (i.e., get chunk_id from previous run) - startdate_sim_yyyymmddhh_prev = ( - cfg.enddate_sim - - timedelta(hours=2 * cfg.restart_step_hours + - cfg.spinup)).strftime('%Y%m%d%H') - - cfg.chunk_id_prev = f'{startdate_sim_yyyymmddhh_prev}_{enddate_sim_yyyymmddhh_prev}' - cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev - cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' - - # No restart for spinup simulations (= default values for no restart) + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev + cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' cfg.cosmo_restart_out = '' cfg.cosmo_restart_in = '' elif 'restart' in cfg.workflow['features']: - cfg.startdate_sim_prev = cfg.startdate_sim - timedelta( - hours=cfg.restart_step_hours) - cfg.enddate_sim_prev = cfg.enddate_sim - timedelta( - hours=cfg.restart_step_hours) - cfg.startdate_sim_prev_yyyymmddhh = cfg.startdate_sim_prev.strftime( - '%Y%m%d%H') - cfg.enddate_sim_prev_yyyymmddhh = cfg.enddate_sim_prev.strftime( - '%Y%m%d%H') - - cfg.chunk_id_prev = f'{cfg.startdate_sim_prev_yyyymmddhh}_{cfg.enddate_sim_prev_yyyymmddhh}' cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev - - # Set restart directories cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' - # Check constraint - if hasattr(cfg, 'constraint'): - assert cfg.constraint in ['gpu', 'mc'], ("Unknown constraint, use" - "gpu or mc") - - # If nested run: use output of mother-simulation - if 'nesting' in cfg.workflow['features'] and not os.path.isdir( - cfg.meteo.dir): - # if ifs_hres_dir doesn't point to a directory, - # it is the name of the mother run - mother_name = cfg.meteo.dir - cfg.meteo.dir = cfg.work_root / mother_name / cfg.chunk_id / 'cosmo' / 'output' - cfg.meteo.inc = 1 - cfg.meteo.prefix = 'lffd' - - # Logging - cfg.log_working_dir = cfg.chain_root / 'checkpoints' / 'working' - cfg.log_finished_dir = cfg.chain_root / 'checkpoints' / 'finished' - - # Create working directories - tools.create_dir(cfg.chain_root, "chain_root") - tools.create_dir(cfg.log_working_dir, "log_working") - tools.create_dir(cfg.log_finished_dir, "log_finished") - - # Number of levels and switch for unit conversion for 'reduce_output' job - if not hasattr(cfg, 'output_levels'): - cfg.output_levels = -1 - if not hasattr(cfg, 'convert_gas'): - cfg.convert_gas = True if cfg.is_async: # Empty curent job ids @@ -433,6 +373,11 @@ def main(): if args.force_sync: cfg.is_async = None + # Check constraint + if cfg.constraint: + assert cfg.constraint in ['gpu', 'mc'], ("Unknown constraint, use" + "gpu or mc") + # Print config before chain starts cfg.print_config() From c9846beaa2155156addc2b67422a9cc562049efc Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 30 Jan 2024 16:56:18 +0000 Subject: [PATCH 229/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_cosmo.py | 1 + jobs/reduce_output.py | 1 - run_chain.py | 3 +-- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jobs/prepare_cosmo.py b/jobs/prepare_cosmo.py index f0208333..8cea9005 100644 --- a/jobs/prepare_cosmo.py +++ b/jobs/prepare_cosmo.py @@ -62,6 +62,7 @@ def set_cfg_variables(cfg): cfg.meteo.inc = 1 cfg.meteo.prefix = 'lffd' + def main(cfg): """ **COSMO Data Preparation** diff --git a/jobs/reduce_output.py b/jobs/reduce_output.py index 435dbfeb..af68b2cc 100644 --- a/jobs/reduce_output.py +++ b/jobs/reduce_output.py @@ -75,7 +75,6 @@ def main(cfg): cfg.output_levels = -1 if not hasattr(cfg, 'convert_gas'): cfg.convert_gas = True - """Get list of constant files""" cfiles = [] read_cfile = False diff --git a/run_chain.py b/run_chain.py index 0f7e5a72..1d1f4d36 100755 --- a/run_chain.py +++ b/run_chain.py @@ -161,7 +161,6 @@ def run_chunk(cfg, force, resume): cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' - if cfg.is_async: # Empty curent job ids cfg.job_ids['current'] = {} @@ -376,7 +375,7 @@ def main(): # Check constraint if cfg.constraint: assert cfg.constraint in ['gpu', 'mc'], ("Unknown constraint, use" - "gpu or mc") + "gpu or mc") # Print config before chain starts cfg.print_config() From 8f700a119733cd4fd1be541145b61382ab745d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 18:06:57 +0100 Subject: [PATCH 230/245] Fix for chunk_id_prev --- run_chain.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/run_chain.py b/run_chain.py index 0f7e5a72..88d008c9 100755 --- a/run_chain.py +++ b/run_chain.py @@ -152,12 +152,14 @@ def run_chunk(cfg, force, resume): # Config variables for spinup and restart runs if hasattr(cfg, 'spinup'): - cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev - cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' + if cfg.chunk_id_prev: + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev + cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' cfg.cosmo_restart_out = '' cfg.cosmo_restart_in = '' elif 'restart' in cfg.workflow['features']: - cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev + if cfg.chunk_id_prev: + cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' From f223a7c876ea49168c049cdfcb7e436d0cdb3ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Tue, 30 Jan 2024 18:15:48 +0100 Subject: [PATCH 231/245] Fix --- run_chain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_chain.py b/run_chain.py index c9eb994b..ff9d174f 100755 --- a/run_chain.py +++ b/run_chain.py @@ -152,13 +152,13 @@ def run_chunk(cfg, force, resume): # Config variables for spinup and restart runs if hasattr(cfg, 'spinup'): - if cfg.chunk_id_prev: + if hasattr(cfg, 'chunk_id_prev'): cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' cfg.cosmo_restart_out = '' cfg.cosmo_restart_in = '' elif 'restart' in cfg.workflow['features']: - if cfg.chunk_id_prev: + if hasattr(cfg, 'chunk_id_prev'): cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' From 3c79da129c32247f105d87fe92dc536f32d2be3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 09:54:22 +0100 Subject: [PATCH 232/245] Save total chunk list --- config.py | 10 +++++----- run_chain.py | 12 +++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index e418828b..1116d6c7 100644 --- a/config.py +++ b/config.py @@ -337,8 +337,8 @@ def format_duration(self, duration): formatted_duration = f"{int(days)}d {int(hours)}h {int(minutes)}m {int(seconds)}s" return formatted_duration - def get_chunks(self): - self.chunks = [] + def get_chunk_list(self): + self.chunk_list = [] for startdate_sim in tools.iter_hours(self.startdate, self.enddate, self.restart_step_hours): if 'spinup' in self.workflow['features'] and hasattr( @@ -356,14 +356,14 @@ def get_chunks(self): if enddate_sim > self.enddate: continue - self.chunks.append(chunk_id) + self.chunk_list.append(chunk_id) def get_previous_chunk_id(self, current_chunk_id): """Get the previous chunk ID based on the current chunk ID.""" try: - index = self.chunks.index(current_chunk_id) + index = self.chunk_list.index(current_chunk_id) if index > 0: - self.chunk_id_prev = self.chunks[index - 1] + self.chunk_id_prev = self.chunk_list[index - 1] except ValueError: # If the current_chunk_id is not in the list or it's the first chunk self.chunk_id_prev = None diff --git a/run_chain.py b/run_chain.py index ff9d174f..686a397d 100755 --- a/run_chain.py +++ b/run_chain.py @@ -289,8 +289,8 @@ def restart_runs(cfg, force, resume): for chunk_id in cfg.chunks: cfg.chunk_id = chunk_id cfg.get_previous_chunk_id(cfg.chunk_id) - cfg.startdate_sim_yyyymmddhh = chunk_id[0:10] - cfg.enddate_sim_yyyymmddhh = chunk_id[-10:] + cfg.startdate_sim_yyyymmddhh = cfg.chunk_id[0:10] + cfg.enddate_sim_yyyymmddhh = cfg.chunk_id[-10:] cfg.startdate_sim = datetime.strptime( cfg.startdate_sim_yyyymmddhh, "%Y%m%d%H").replace(tzinfo=pytz.UTC) cfg.enddate_sim = datetime.strptime( @@ -364,11 +364,9 @@ def main(): else: cfg.jobs = args.job_list - # Check if chunks are set or if all are used and compute them - if args.chunk_list is None: - cfg.get_chunks() - else: - cfg.chunks = args.chunk_list + # Get chunks and custom chunks if present + cfg.get_chunk_list() + cfg.chunks = args.chunk_list if args.chunk_list else cfg.chunk_list # Check sync is forced if args.force_sync: From 03370b5ad168d69d7f2a49943c52cafb5e96aff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 10:30:10 +0100 Subject: [PATCH 233/245] Fix restart variables --- config.py | 10 ++++------ jobs/prepare_icon.py | 5 +++-- run_chain.py | 12 ++++++------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/config.py b/config.py index 1116d6c7..3827f974 100644 --- a/config.py +++ b/config.py @@ -360,12 +360,10 @@ def get_chunk_list(self): def get_previous_chunk_id(self, current_chunk_id): """Get the previous chunk ID based on the current chunk ID.""" - try: - index = self.chunk_list.index(current_chunk_id) - if index > 0: - self.chunk_id_prev = self.chunk_list[index - 1] - except ValueError: - # If the current_chunk_id is not in the list or it's the first chunk + index = self.chunk_list.index(current_chunk_id) + if index > 0: + self.chunk_id_prev = self.chunk_list[index - 1] + else: self.chunk_id_prev = None def get_dep_ids(self, job_name, add_dep=None): diff --git a/jobs/prepare_icon.py b/jobs/prepare_icon.py index f6c6ac1b..ca3d6872 100644 --- a/jobs/prepare_icon.py +++ b/jobs/prepare_icon.py @@ -16,8 +16,9 @@ def set_cfg_variables(cfg): cfg.icon_output = cfg.icon_base / 'output' cfg.icon_output_reduced = cfg.icon_base / 'output_reduced' cfg.icon_restart_out = cfg.icon_base / 'restart' - cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' - cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' + if cfg.chunk_id_prev: + cfg.icon_restart_in = cfg.chain_root_prev / 'icon' / 'run' + cfg.icon_input_icbc_prev = cfg.chain_root_prev / 'icon' / 'input' / 'icbc' cfg.input_files_scratch = {} for dsc, file in cfg.input_files.items(): diff --git a/run_chain.py b/run_chain.py index 686a397d..bccd241a 100755 --- a/run_chain.py +++ b/run_chain.py @@ -151,17 +151,17 @@ def run_chunk(cfg, force, resume): tools.create_dir(cfg.log_finished_dir, "log_finished") # Config variables for spinup and restart runs + cfg.cosmo_restart_in = '' + cfg.cosmo_restart_out = '' if hasattr(cfg, 'spinup'): - if hasattr(cfg, 'chunk_id_prev'): + if cfg.chunk_id_prev: cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev cfg.last_cosmo_output = cfg.chain_root_prev / 'cosmo' / 'output' - cfg.cosmo_restart_out = '' - cfg.cosmo_restart_in = '' elif 'restart' in cfg.workflow['features']: - if hasattr(cfg, 'chunk_id_prev'): + if cfg.chunk_id_prev: cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev - cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' - cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' + cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' + cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' if cfg.is_async: # Empty curent job ids From 45eec53ada47b384208868f43c6eb9c004db5a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 10:37:16 +0100 Subject: [PATCH 234/245] Don't print chunk list twice --- run_chain.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/run_chain.py b/run_chain.py index bccd241a..1080d54c 100755 --- a/run_chain.py +++ b/run_chain.py @@ -364,10 +364,6 @@ def main(): else: cfg.jobs = args.job_list - # Get chunks and custom chunks if present - cfg.get_chunk_list() - cfg.chunks = args.chunk_list if args.chunk_list else cfg.chunk_list - # Check sync is forced if args.force_sync: cfg.is_async = None @@ -377,9 +373,15 @@ def main(): assert cfg.constraint in ['gpu', 'mc'], ("Unknown constraint, use" "gpu or mc") + # Get complete chunk list + cfg.get_chunk_list() + # Print config before chain starts cfg.print_config() + # Get custom chunks if specified + cfg.chunks = args.chunk_list if args.chunk_list else cfg.chunk_list + tools.create_dir(cfg.case_root, "case_root") print("╔════════════════════════════════════════╗") From 5174a9217da7fd221736fdb514c974979dc99116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 11:17:36 +0100 Subject: [PATCH 235/245] Fix for cosmo_restart_out --- run_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_chain.py b/run_chain.py index 1080d54c..377ad3e9 100755 --- a/run_chain.py +++ b/run_chain.py @@ -161,7 +161,7 @@ def run_chunk(cfg, force, resume): if cfg.chunk_id_prev: cfg.chain_root_prev = cfg.work_root / cfg.casename / cfg.chunk_id_prev cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' - cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' + cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' if cfg.is_async: # Empty curent job ids From 7505cc02225f7fef6ed07f02e223af86e2bbc41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 14:22:02 +0100 Subject: [PATCH 236/245] Remove sequential part --- run_chain.py | 111 ++++++++++++++++++--------------------------------- 1 file changed, 39 insertions(+), 72 deletions(-) diff --git a/run_chain.py b/run_chain.py index 377ad3e9..794da8e7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -163,7 +163,7 @@ def run_chunk(cfg, force, resume): cfg.cosmo_restart_in = cfg.chain_root_prev / 'cosmo' / 'restart' cfg.cosmo_restart_out = cfg.chain_root / 'cosmo' / 'restart' - if cfg.is_async: + if not cfg.force_sync: # Empty curent job ids cfg.job_ids['current'] = {} @@ -191,77 +191,41 @@ def run_chunk(cfg, force, resume): # Wait for previous chunk jobs, monitor them and cycle info cfg.cycle() - else: - # run jobs (if required) - for job in cfg.jobs: - skip = False - - # if exists job is currently worked on or has been finished - if (cfg.log_working_dir / job).exists(): - if not force: - while True: - if (cfg.log_finished_dir / job).exists(): - print( - f' └── Skip "{job}" for chunk "{cfg.chunk_id}"' - ) - skip = True - break - elif resume: - resume = False - break - else: - print( - f" └── Wait for {job} of chunk {cfg.chunk_id}" - ) - sys.stdout.flush() - for _ in range(3000): - time.sleep(0.1) - else: - (cfg.log_working_dir / job).unlink() - (cfg.log_finished_dir / job).unlink(missing_ok=True) - - if not skip: - print(f' └── Process "{job}" for chunk "{cfg.chunk_id}"') - sys.stdout.flush() - - try_count = 1 + (cfg.ntry - 1) * (job == 'cosmo') - while try_count > 0: - try_count -= 1 - try: - # Change the log file - cfg.logfile = cfg.log_working_dir / job - cfg.logfile_finish = cfg.log_finished_dir / job - - # Launch the job - to_call = getattr(jobs, job) - to_call.main(cfg) - - shutil.copy(cfg.logfile, cfg.logfile_finish) - - exitcode = 0 - try_count = 0 - except Exception: - subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( - job, cfg.chunk_id) - logging.exception(subject) - if cfg.user_mail: - message = tools.prepare_message( - cfg.log_working_dir / job) - logging.info('Sending log file to %s' % - cfg.user_mail) - tools.send_mail(cfg.user_mail, subject, message) - if try_count == 0: - raise RuntimeError(subject) - - if exitcode != 0 or not (cfg.log_finished_dir / job).exists(): - subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( - job, cfg.chunk_id) - if cfg.user_mail: - message = tools.prepare_message(cfg.log_working_dir / - job) - logging.info('Sending log file to %s' % cfg.user_mail) - tools.send_mail(cfg.user_mail, subject, message) - raise RuntimeError(subject) + else: # For nested run_chain.py + for job_name in cfg.jobs: + print(f' └── Process "{job_name}" for chunk "{cfg.chunk_id}"') + try: + # Change the log file + cfg.logfile = cfg.log_working_dir / job_name + cfg.logfile_finish = cfg.log_finished_dir / job_name + + # Launch the job + to_call = getattr(jobs, job_name) + to_call.main(cfg) + + shutil.copy(cfg.logfile, cfg.logfile_finish) + + exitcode = 0 + except Exception: + subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( + job_name, cfg.chunk_id) + logging.exception(subject) + if cfg.user_mail: + message = tools.prepare_message( + cfg.log_working_dir / job_name) + logging.info('Sending log file to %s' % + cfg.user_mail) + tools.send_mail(cfg.user_mail, subject, message) + + if exitcode != 0 or not (cfg.log_finished_dir / job_name).exists(): + subject = "ERROR or TIMEOUT in job '%s' for chain '%s'" % ( + job_name, cfg.chunk_id) + if cfg.user_mail: + message = tools.prepare_message(cfg.log_working_dir / + job_name) + logging.info('Sending log file to %s' % cfg.user_mail) + tools.send_mail(cfg.user_mail, subject, message) + raise RuntimeError(subject) def restart_runs(cfg, force, resume): @@ -367,6 +331,9 @@ def main(): # Check sync is forced if args.force_sync: cfg.is_async = None + cfg.force_sync = True + else: + cfg.force_sync = False # Check constraint if cfg.constraint: From 951ad5251c96fe5cfdbb5d173a926fe370a91085 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 31 Jan 2024 13:22:29 +0000 Subject: [PATCH 237/245] GitHub Action: Apply Pep8-formatting --- run_chain.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/run_chain.py b/run_chain.py index 794da8e7..33cb4cf7 100755 --- a/run_chain.py +++ b/run_chain.py @@ -191,7 +191,7 @@ def run_chunk(cfg, force, resume): # Wait for previous chunk jobs, monitor them and cycle info cfg.cycle() - else: # For nested run_chain.py + else: # For nested run_chain.py for job_name in cfg.jobs: print(f' └── Process "{job_name}" for chunk "{cfg.chunk_id}"') try: @@ -211,10 +211,9 @@ def run_chunk(cfg, force, resume): job_name, cfg.chunk_id) logging.exception(subject) if cfg.user_mail: - message = tools.prepare_message( - cfg.log_working_dir / job_name) - logging.info('Sending log file to %s' % - cfg.user_mail) + message = tools.prepare_message(cfg.log_working_dir / + job_name) + logging.info('Sending log file to %s' % cfg.user_mail) tools.send_mail(cfg.user_mail, subject, message) if exitcode != 0 or not (cfg.log_finished_dir / job_name).exists(): From 3fe7fcc753edf0d94694b6bf2fd588e84e582316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 16:09:27 +0100 Subject: [PATCH 238/245] Remove sequential case --- cases/icon-seq-test/config.yaml | 61 ---- cases/icon-seq-test/icon_runjob.cfg | 345 ------------------ .../icontools_remap_00_lbc_runjob.cfg | 154 -------- .../icontools_remap_ic_runjob.cfg | 245 ------------- .../icontools_remap_lbc_rest_runjob.cfg | 128 ------- 5 files changed, 933 deletions(-) delete mode 100644 cases/icon-seq-test/config.yaml delete mode 100755 cases/icon-seq-test/icon_runjob.cfg delete mode 100755 cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg delete mode 100755 cases/icon-seq-test/icontools_remap_ic_runjob.cfg delete mode 100755 cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg diff --git a/cases/icon-seq-test/config.yaml b/cases/icon-seq-test/config.yaml deleted file mode 100644 index a496e575..00000000 --- a/cases/icon-seq-test/config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Configuration file for the 'icon-seq-test' case with ICON - -workflow: - features: - - restart - jobs: - - prepare_icon - - icontools - - icon -constraint: gpu -run_on: cpu -compute_queue: normal -ntasks_per_node: 12 -restart_step: PT6H -startdate: 2018-01-01T00:00:00Z -enddate: 2018-01-01T12:00:00Z - -eccodes_dir: ./input/eccodes_definitions -iconremap_bin: iconremap -iconsub_bin: iconsub -latbc_filename: ifs_201801_lbc.nc -inidata_filename: ifs_init_2018010100.nc -output_filename: NWP_LAM -filename_format: _DOM_ -lateral_boundary_grid_order: lateral_boundary - -walltime: - icon: '00:30:00' - -meteo: - dir: ./input/meteo - prefix: ifs_ - nameformat: '%Y%m%d%H' - suffix: .grb - inc: 3 - -icontools_runjobs: - - icontools_remap_ic_runjob.cfg - - icontools_remap_00_lbc_runjob.cfg - - icontools_remap_lbc_rest_runjob.cfg - -input_files: - radiation_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.parent.nc - dynamics_grid_filename: ./input/icon/grid/VERIFY_DOM_DOM01.nc - map_file_latbc: ./input/icon/grid/map_file.latbc - lateral_boundary_grid: ./input/icon/grid/lateral_boundary.grid.nc - extpar_filename: ./input/icon/grid/external_parameter_icon_VERIFY_DOM_DOM01_tiles.nc - cldopt_filename: ./input/icon/rad/rrtm_cldopt.nc - lrtm_filename: ./input/icon/rad/rrtmg_lw.nc - map_file_ana: ./input/icon/mapping/map_file.ana - -icon: - binary_file: ./ext/icon/bin/icon - runjob_filename: icon_runjob.cfg - compute_queue: normal - walltime: '00:10:00' - np_tot: 8 - np_io: 1 - np_restart: 1 - np_prefetch: 1 - diff --git a/cases/icon-seq-test/icon_runjob.cfg b/cases/icon-seq-test/icon_runjob.cfg deleted file mode 100755 index 07fabaaf..00000000 --- a/cases/icon-seq-test/icon_runjob.cfg +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=icon -#SBATCH --account={cfg.compute_account} -#SBATCH --time={cfg.walltime_icon} -#SBATCH --nodes={cfg.icon_np_tot} -#SBATCH --ntasks-per-node={cfg.ntasks_per_node} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --hint=nomultithread -#SBATCH --output={cfg.logfile} -#SBATCH --open-mode=append -#SBATCH --chdir={cfg.icon_work} - -# OpenMP environment variables -# ---------------------------- -export OMP_NUM_THREADS=1 -export ICON_THREADS=1 -export OMP_SCHEDULE=static,12 -export OMP_DYNAMIC="false" -export OMP_STACKSIZE=200M -# -# MPI variables -# ------------- -no_of_nodes={cfg.icon_np_tot} -mpi_procs_pernode={cfg.ntasks_per_node} -((mpi_total_procs=no_of_nodes * mpi_procs_pernode)) -# -# blocking length -# --------------- -nproma=16 - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -# ---------------------------------------------------------------------------- -# create ICON master namelist -# ---------------------------------------------------------------------------- - -cat > icon_master.namelist << EOF -! master_nml: ---------------------------------------------------------------- -&master_nml - lrestart = {cfg.lrestart} ! .TRUE.=current experiment is resumed - read_restart_namelists = .true. -/ - -! master_time_control_nml: --------------------------------------------------- -&master_time_control_nml - calendar = 'proleptic gregorian' - restartTimeIntval = '{cfg.restart_step}' - checkpointTimeIntval = '{cfg.restart_step}' - experimentStartDate = '{cfg.ini_datetime_string}' - experimentStopDate = '{cfg.end_datetime_string}' -/ - -! master_model_nml: repeated for each model ---------------------------------- -&master_model_nml - model_type = 1 ! identifies which component to run (atmosphere,ocean,...) - model_name = "ATMO" ! character string for naming this component. - model_namelist_filename = "NAMELIST_{cfg.casename}" ! file name containing the model namelists - model_min_rank = 1 ! start MPI rank for this model - model_max_rank = 65536 ! end MPI rank for this model - model_inc_rank = 1 ! stride of MPI ranks -/ -EOF - -# ---------------------------------------------------------------------- -# model namelists -# ---------------------------------------------------------------------- - -cat > NAMELIST_{cfg.casename} << EOF -! parallel_nml: MPI parallelization ------------------------------------------- -¶llel_nml - nproma = 128 ! loop chunk length - p_test_run = .FALSE. ! .TRUE. means verification run for MPI parallelization - num_io_procs = {cfg.icon_np_io} ! number of I/O processors - num_restart_procs = {cfg.icon_np_restart} ! number of restart processors - num_prefetch_proc = {cfg.icon_np_prefetch} ! number of processors for LBC prefetching - iorder_sendrecv = 3 ! sequence of MPI send/receive calls -/ - - -! run_nml: general switches --------------------------------------------------- -&run_nml - ltestcase = .FALSE. ! real case run - num_lev = 60 ! number of full levels (atm.) for each domain - lvert_nest = .FALSE. ! no vertical nesting - dtime = 60. ! timestep in seconds - ldynamics = .TRUE. ! compute adiabatic dynamic tendencies - ltransport = .TRUE. ! compute large-scale tracer transport - ntracer = 0 ! number of advected tracers - iforcing = 3 ! forcing of dynamics and transport by parameterized processes - msg_level = 13 ! detailed report during integration - ltimer = .TRUE. ! timer for monitoring the runtime of specific routines - timers_level = 10 ! performance timer granularity - check_uuid_gracefully = .TRUE. ! give only warnings for non-matching uuids - output = "nml" ! main switch for enabling/disabling components of the model output - lart = .FALSE. ! main switch for ART - debug_check_level = 10 - restart_filename = "{cfg.icon_restart_out}/{cfg.output_filename}_.nc" - activate_sync_timers = .TRUE. -/ - -! diffusion_nml: horizontal (numerical) diffusion ---------------------------- -&diffusion_nml - lhdiff_vn = .TRUE. ! diffusion on the horizontal wind field - lhdiff_temp = .TRUE. ! diffusion on the temperature field - lhdiff_w = .TRUE. ! diffusion on the vertical wind field - hdiff_order = 5 ! order of nabla operator for diffusion - itype_vn_diffu = 1 ! reconstruction method used for Smagorinsky diffusion - itype_t_diffu = 2 ! discretization of temperature diffusion - hdiff_efdt_ratio = 24.0 ! ratio of e-folding time to time step - hdiff_smag_fac = 0.025 ! scaling factor for Smagorinsky diffusion -/ - -! dynamics_nml: dynamical core ----------------------------------------------- -&dynamics_nml - iequations = 3 ! type of equations and prognostic variables - idiv_method = 1 ! method for divergence computation - divavg_cntrwgt = 0.50 ! weight of central cell for divergence averaging - lcoriolis = .TRUE. ! Coriolis force -/ - -! extpar_nml: external data -------------------------------------------------- -&extpar_nml - itopo = 1 ! topography (0:analytical) - extpar_filename = '{cfg.input_files_scratch_extpar_filename}' ! filename of external parameter input file - n_iter_smooth_topo = 1,1 ! iterations of topography smoother - heightdiff_threshold = 3000. ! height difference between neighb. grid points - hgtdiff_max_smooth_topo = 750.,750. ! see Namelist doc - heightdiff_threshold = 2250.,1500. -/ - -! initicon_nml: specify read-in of initial state ------------------------------ -&initicon_nml - init_mode = 2 ! 7: start from DWD fg with subsequent vertical remapping - lread_ana = .FALSE. ! no analysis data will be read - ifs2icon_filename = "{inidata_filename}" ! initial data filename - ana_varnames_map_file = "{cfg.input_files_scratch_map_file_ana}" ! dictionary mapping internal names onto GRIB2 shortNames - ltile_coldstart = .TRUE. ! coldstart for surface tiles - ltile_init = .FALSE. ! set it to .TRUE. if FG data originate from run without tiles -/ - -! grid_nml: horizontal grid -------------------------------------------------- -&grid_nml - dynamics_grid_filename = "{cfg.input_files_scratch_dynamics_grid_filename}" ! array of the grid filenames for the dycore - radiation_grid_filename = "{cfg.input_files_scratch_radiation_grid_filename}" ! array of the grid filenames for the radiation model - dynamics_parent_grid_id = 0 ! array of the indexes of the parent grid filenames - lredgrid_phys = .TRUE. ! .true.=radiation is calculated on a reduced grid - lfeedback = .TRUE. ! specifies if feedback to parent grid is performed - l_limited_area = .TRUE. ! .TRUE. performs limited area run - ifeedback_type = 2 ! feedback type (incremental/relaxation-based) - start_time = 0. ! Time when a nested domain starts to be active [s] -/ - -! gridref_nml: grid refinement settings -------------------------------------- -&gridref_nml - denom_diffu_v = 150. ! denominator for lateral boundary diffusion of velocity -/ - -! interpol_nml: settings for internal interpolation methods ------------------ -&interpol_nml - nudge_zone_width = 8 ! width of lateral boundary nudging zone - support_baryctr_intp = .FALSE. ! barycentric interpolation support for output - nudge_max_coeff = 0.07 - nudge_efold_width = 2.0 -/ - -! io_nml: general switches for model I/O ------------------------------------- -&io_nml - itype_pres_msl = 5 ! method for computation of mean sea level pressure - itype_rh = 1 ! method for computation of relative humidity - lmask_boundary = .TRUE. ! mask out interpolation zone in output - restart_file_type = 5 -/ - -! limarea_nml: settings for limited area mode --------------------------------- -&limarea_nml - itype_latbc = 1 ! 1: time-dependent lateral boundary conditions - dtime_latbc = 10800 ! time difference between 2 consecutive boundary data - nlev_latbc = 90 ! Number of vertical levels in boundary data - latbc_boundary_grid = "{cfg.input_files_scratch_lateral_boundary_grid}" ! Grid file defining the lateral boundary - latbc_path = "{cfg.icon_input_icbc}" ! Absolute path to boundary data - latbc_varnames_map_file = "{cfg.input_files_scratch_map_file_latbc}" - latbc_filename = "{cfg.latbc_filename}" ! boundary data input filename - init_latbc_from_fg = .FALSE. ! .TRUE.: take lbc for initial time from first guess -/ - -! lnd_nml: land scheme switches ----------------------------------------------- -&lnd_nml - ntiles = 1 ! number of tiles - nlev_snow = 3 ! number of snow layers - lmulti_snow = .FALSE. ! .TRUE. for use of multi-layer snow model - idiag_snowfrac = 20 ! type of snow-fraction diagnosis - lsnowtile = .TRUE. ! .TRUE.=consider snow-covered and snow-free separately - itype_root = 2 ! root density distribution - itype_heatcond = 3 ! type of soil heat conductivity - itype_lndtbl = 4 ! table for associating surface parameters - itype_evsl = 4 ! type of bare soil evaporation - itype_root = 2 ! root density distribution - cwimax_ml = 5.e-4 ! scaling parameter for max. interception storage - c_soil = 1.75 ! surface area density of the evaporative soil surface - c_soil_urb = 0.5 ! same for urban areas - lseaice = .TRUE. ! .TRUE. for use of sea-ice model - llake = .TRUE. ! .TRUE. for use of lake model -/ - -! nonhydrostatic_nml: nonhydrostatic model ----------------------------------- -&nonhydrostatic_nml - iadv_rhotheta = 2 ! advection method for rho and rhotheta - ivctype = 2 ! type of vertical coordinate - itime_scheme = 4 ! time integration scheme - ndyn_substeps = 5 ! number of dynamics steps per fast-physics step - exner_expol = 0.333 ! temporal extrapolation of Exner function - vwind_offctr = 0.2 ! off-centering in vertical wind solver - damp_height = 12500.0 ! height at which Rayleigh damping of vertical wind starts - rayleigh_coeff = 1.5 ! Rayleigh damping coefficient - divdamp_order = 24 ! order of divergence damping - divdamp_type = 3 ! type of divergence damping - divdamp_fac = 0.004 ! scaling factor for divergence damping - l_open_ubc = .FALSE. ! .TRUE.=use open upper boundary condition - igradp_method = 3 ! discretization of horizontal pressure gradient - l_zdiffu_t = .TRUE. ! specifies computation of Smagorinsky temperature diffusion - thslp_zdiffu = 0.02 ! slope threshold (temperature diffusion) - thhgtd_zdiffu = 125.0 ! threshold of height difference (temperature diffusion) - htop_moist_proc = 22500.0 ! max. height for moist physics - hbot_qvsubstep = 22500.0 ! height above which QV is advected with substepping scheme -/ - -! nwp_phy_nml: switches for the physics schemes ------------------------------ -&nwp_phy_nml - inwp_gscp = 2 ! cloud microphysics and precipitation - inwp_convection = 1 ! convection - lshallowconv_only = .FALSE. ! only shallow convection - inwp_radiation = 1 ! radiation - inwp_cldcover = 1 ! cloud cover scheme for radiation - inwp_turb = 1 ! vertical diffusion and transfer - inwp_satad = 1 ! saturation adjustment - inwp_sso = 1 ! subgrid scale orographic drag - inwp_gwd = 0 ! non-orographic gravity wave drag - inwp_surface = 1 ! surface scheme - latm_above_top = .TRUE. ! take into account atmosphere above model top for radiation computation - ldetrain_conv_prec = .TRUE. - efdt_min_raylfric = 7200. ! minimum e-folding time of Rayleigh friction - itype_z0 = 2 ! type of roughness length data - icapdcycl = 3 ! apply CAPE modification to improve diurnalcycle over tropical land - icpl_aero_conv = 1 ! coupling between autoconversion and Tegen aerosol climatology - icpl_aero_gscp = 1 ! coupling between autoconversion and Tegen aerosol climatology - lrtm_filename = '{cfg.input_files_scratch_lrtm_filename}' ! longwave absorption coefficients for RRTM_LW - cldopt_filename = '{cfg.input_files_scratch_cldopt_filename}' ! RRTM cloud optical properties - dt_rad = 720. ! time step for radiation in s - dt_conv = 120.,90.,90. ! time step for convection in s (domain specific) - dt_sso = 120.,360.,360. ! time step for SSO parameterization - dt_gwd = 360.,360.,360. ! time step for gravity wave drag parameterization -/ - -! nwp_tuning_nml: additional tuning parameters ---------------------------------- -&nwp_tuning_nml - itune_albedo = 1 ! reduced albedo (w.r.t. MODIS data) over Sahara - tune_gkwake = 1.8 - tune_gkdrag = 0.01 - tune_minsnowfrac = 0.3 -/ - -! output_nml: specifies an output stream -------------------------------------- -&output_nml - filetype = 4 ! output format: 2=GRIB2, 4=NETCDFv2 - dom = 1 ! write domain 1 only - output_bounds = 0., 10000000., 3600. ! start, end, increment - steps_per_file = 1 ! number of steps per file - mode = 1 ! 1: forecast mode (relative t-axis), 2: climate mode (absolute t-axis) - include_last = .TRUE. - output_filename = '{cfg.output_filename}' - filename_format = '{cfg.icon_output}/_DOM_' ! file name base - steps_per_file_inclfirst = .FALSE. - output_grid = .TRUE. - remap = 1 ! 1: remap to lat-lon grid - !north_pole = -170.,40. ! definition of north_pole for rotated lat-lon grid - reg_lon_def = -16.0,0.13,36.0 ! - reg_lat_def = 32.0,0.12,74.0 ! - ml_varlist = 'group:PBL_VARS', - 'group:ATMO_ML_VARS', - 'group:precip_vars', - 'group:land_vars', - 'group:nh_prog_vars', - 'z_mc', 'z_ifc', -/ - -! radiation_nml: radiation scheme --------------------------------------------- -&radiation_nml - irad_o3 = 7 ! ozone climatology - irad_aero = 6 ! aerosols - albedo_type = 2 ! type of surface albedo - vmr_co2 = 390.e-06 - vmr_ch4 = 1800.e-09 - vmr_n2o = 322.0e-09 - vmr_o2 = 0.20946 - vmr_cfc11 = 240.e-12 - vmr_cfc12 = 532.e-12 -/ - -! sleve_nml: vertical level specification ------------------------------------- -&sleve_nml - min_lay_thckn = 20.0 ! layer thickness of lowermost layer - top_height = 23000.0 ! height of model top - stretch_fac = 0.65 ! stretching factor to vary distribution of model levels - decay_scale_1 = 4000.0 ! decay scale of large-scale topography component - decay_scale_2 = 2500.0 ! decay scale of small-scale topography component - decay_exp = 1.2 ! exponent of decay function - flat_height = 16000.0 ! height above which the coordinate surfaces are flat -/ - -! transport_nml: tracer transport --------------------------------------------- -&transport_nml - npassive_tracer = 0 ! number of additional passive tracers - ivadv_tracer = 3, 3, 3, 3, 3, 3 ! tracer specific method to compute vertical advection - itype_hlimit = 3, 4, 4, 4, 4, 4 ! type of limiter for horizontal transport - ihadv_tracer = 52, 2, 2, 2, 2, 22 ! tracer specific method to compute horizontal advection - llsq_svd = .TRUE. ! use SV decomposition for least squares design matrix -/ - -! turbdiff_nml: turbulent diffusion ------------------------------------------- -&turbdiff_nml - tkhmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient - tkmmin = 0.75 ! scaling factor for minimum vertical diffusion coefficient - pat_len = 750.0 ! effective length scale of thermal surface patterns - c_diff = 0.2 ! length scale factor for vertical diffusion of TKE - rat_sea = 7.5 ! controls laminar resistance for sea surface - rlam_heat = 1.5 - ltkesso = .TRUE. ! consider TKE-production by sub-grid SSO wakes - frcsmot = 0.2 ! these 2 switches together apply vertical smoothing of the TKE source terms - imode_frcsmot = 2 ! in the tropics (only), which reduces the moist bias in the tropical lower troposphere - itype_sher = 3 ! type of shear forcing used in turbulence - ltkeshs = .TRUE. ! include correction term for coarse grids in hor. shear production term - a_hshr = 2.0 ! length scale factor for separated horizontal shear mode - icldm_turb = 1 ! mode of cloud water representation in turbulence - ldiff_qi = .TRUE. -/ - -EOF - -# ---------------------------------------------------------------------- -# run the model! -# ---------------------------------------------------------------------- - srun ./icon.exe diff --git a/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg b/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg deleted file mode 100755 index 1f454071..00000000 --- a/cases/icon-seq-test/icontools_remap_00_lbc_runjob.cfg +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=iconsub -#SBATCH --account={cfg.compute_account} -#SBATCH --chdir={cfg.icon_work} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --nodes=1 -#SBATCH --ntasks-per-core=1 -#SBATCH --ntasks-per-node={cfg.ntasks_per_node} -#SBATCH --cpus-per-task=1 -#SBATCH --output={logfile} -#SBATCH --open-mode=append - -ulimit -s unlimited - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh -spack load icontools - -#----------------------------------------------------------------------------- -# PART I: Create auxiliary grid file which contains only the cells of the -# boundary zone. -#----------------------------------------------------------------------------- -cat > NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} << EOF_1 -&iconsub_nml - grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}', - output_type = 4, - lwrite_grid = .TRUE., -/ -&subarea_nml - ORDER = "{cfg.lateral_boundary_grid_order}", - grf_info_file = '{cfg.input_files_scratch_dynamics_grid_filename}', - min_refin_c_ctrl = 1 - max_refin_c_ctrl = 14 -/ -EOF_1 - -srun -n 1 {cfg.iconsub_bin} \ - --nml NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} 2>&1 - -#----------------------------------------------------------------------------- -# PART II: Extract boundary data -#----------------------------------------------------------------------------- -rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* - -set +x -cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF -! -&input_field_nml ! temperature - inputname = "T" - outputname = "T" - code = 130 - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "U" - outputname = "U" - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "V" - outputname = "V" - intp_method = 3 -/ -&input_field_nml ! vertical velocity - inputname = "OMEGA" - outputname = "W" - code = 135 - intp_method = 3 -/ -&input_field_nml ! surface pressure - inputname = "LNSP" - outputname = "LNPS" - code = 152 - intp_method = 3 -/ -&input_field_nml ! geopotential - inputname = "Z" - outputname = "GEOSP" - code = 129 - intp_method = 3 -/ -&input_field_nml ! specific humidity - inputname = "QV" - outputname = "QV" - code = 133 - intp_method = 3 -/ -&input_field_nml ! cloud liquid water content - inputname = "CLWC" - outputname = "QC" - code = 246 - intp_method = 3 -/ -&input_field_nml ! cloud ice water content - inputname = "CIWC" - outputname = "QI" - code = 247 - intp_method = 3 -/ -&input_field_nml ! rain water content - inputname = "CRWC" - outputname = "QR" - code = 75 - intp_method = 3 -/ -&input_field_nml ! snow water content - inputname = "CSWC" - outputname = "QS" - code = 76 - intp_method = 3 -/ -EOF - -#----------------------------------------------------------------------------- -# loop over file list: - -echo "DATAFILELIST is {datafile_list}" -for datafilename in {datafile_list} ; do - datafile="${{datafilename##*/}}" # get filename without path - outdatafile=${{datafile%.*}} # get filename without suffix - cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C -&remap_nml - in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' - in_filename = '{cfg.meteo_dir}/${{datafile}}' - in_type = 1 - out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' - out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' - out_type = 2 - out_filetype = 4 - l_have3dbuffer = .false. -! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" -/ -EOF_2C - - srun -n 1 {cfg.iconremap_bin} -q \ - --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ - --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 - -done - -#----------------------------------------------------------------------------- -# clean-up - -rm -f nml.log -rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} - -#----------------------------------------------------------------------------- -exit -#----------------------------------------------------------------------------- - diff --git a/cases/icon-seq-test/icontools_remap_ic_runjob.cfg b/cases/icon-seq-test/icontools_remap_ic_runjob.cfg deleted file mode 100755 index e704afb8..00000000 --- a/cases/icon-seq-test/icontools_remap_ic_runjob.cfg +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=iconremap_ic -#SBATCH --account={cfg.compute_account} -#SBATCH --chdir={cfg.icon_work} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --nodes=1 -#SBATCH --ntasks-per-core=1 -#SBATCH --ntasks-per-node={cfg.ntasks_per_node} -#SBATCH --cpus-per-task=1 -#SBATCH --output={logfile} -#SBATCH --open-mode=append - -ulimit -s unlimited - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh -spack load icontools - -#----------------------------------------------------------------------------- -# Remap inital data onto local (limited-area) grid -#----------------------------------------------------------------------------- -cat > NAMELIST_ICONREMAP_FIELDS << EOF -! -&input_field_nml ! temperature - inputname = "T" - outputname = "T" - code = 130 - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "U" - outputname = "U" - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "V" - outputname = "V" - intp_method = 3 -/ -&input_field_nml ! vertical velocity - inputname = "OMEGA" - outputname = "W" - code = 135 - intp_method = 3 -/ -&input_field_nml ! surface pressure - inputname = "LNSP" - outputname = "LNPS" - code = 152 - intp_method = 3 -/ -&input_field_nml ! geopotential - inputname = "Z" - outputname = "GEOSP" - code = 129 - intp_method = 3 -/ -&input_field_nml ! geopotential - inputname = "Z" - outputname = "GEOP_SFC" - code = 129 - intp_method = 3 -/ -&input_field_nml ! specific humidity - inputname = "QV" - outputname = "QV" - code = 133 - intp_method = 3 -/ -&input_field_nml ! cloud liquid water content - inputname = "CLWC" - outputname = "QC" - code = 246 - intp_method = 3 -/ -&input_field_nml ! cloud ice water content - inputname = "CIWC" - outputname = "QI" - code = 247 - intp_method = 3 -/ -&input_field_nml ! rain water content - inputname = "CRWC" - outputname = "QR" - code = 75 - intp_method = 3 -/ -&input_field_nml ! snow water content - inputname = "CSWC" - outputname = "QS" - code = 76 - intp_method = 3 -/ -&input_field_nml ! snow temperature - inputname = "TSN" - outputname = "T_SNOW" - code = 238 - intp_method = 3 -/ -&input_field_nml ! water content of snow - inputname = "SD" - outputname = "W_SNOW" - code = 141 - intp_method = 3 -/ -&input_field_nml ! density of snow - inputname = "RSN" - outputname = "RHO_SNOW" - code = 33 - intp_method = 3 -/ -&input_field_nml ! snow albedo - inputname = "ASN" - outputname = "ALB_SNOW" - code = 32 - intp_method = 3 -/ -&input_field_nml ! skin temperature - inputname = "SKT" - outputname = "SKT" - code = 235 - intp_method = 3 -/ -&input_field_nml ! sea surface temperature - inputname = "SST" - outputname = "SST" - code = 34 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 1 - inputname = "STL1" - outputname = "STL1" - code = 139 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 2 - inputname = "STL2" - outputname = "STL2" - code = 170 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 3 - inputname = "STL3" - outputname = "STL3" - code = 183 - intp_method = 3 -/ -&input_field_nml ! soil temperature level 4 - inputname = "STL4" - outputname = "STL4" - code = 236 - intp_method = 3 -/ -&input_field_nml ! sea-ice cover - inputname = "CI" - outputname = "CI" - code = 31 - intp_method = 3 -/ -&input_field_nml ! water cont. of interception storage - inputname = "SRC" - outputname = "W_I" - code = 198 - intp_method = 3 -/ -&input_field_nml ! surface roughness - inputname = "SR" - outputname = "Z0" - code = 173 - intp_method = 3 -/ -&input_field_nml ! Land/sea mask - inputname = "LSM" - outputname = "LSM" - code = 172 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 1 - inputname = "SWVL1" - outputname = "SMIL1" - code = 80 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 2 - inputname = "SWVL2" - outputname = "SMIL2" - code = 81 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 3 - inputname = "SWVL3" - outputname = "SMIL3" - code = 82 - intp_method = 3 -/ -&input_field_nml ! soil moisture index layer 4 - inputname = "SWVL4" - outputname = "SMIL4" - code = 83 - intp_method = 3 -/ -EOF - -#----------------------------------------------------------------------------- -# loop over file list: - -datafilename={cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix} -datafile="${{datafilename##*/}}" # get filename without path -outdatafile=${{datafile%.*}} # get filename without suffix - -# create ICON master namelist -# ------------------------ -# For a complete list see Namelist_overview and Namelist_overview.pdf - -cat > NAMELIST_ICONREMAP << EOF -&remap_nml - in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' - in_filename = '{cfg.meteo_dir}/${{datafile}}' - in_type = 1 - out_grid_filename = '{cfg.input_files_scratch_dynamics_grid_filename}' - out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}.nc' - out_type = 2 - out_filetype = 4 - l_have3dbuffer = .false. -/ -EOF - -srun -n 1 {cfg.iconremap_bin} \ - -vvvvv -q --remap_nml NAMELIST_ICONREMAP \ - --input_field_nml NAMELIST_ICONREMAP_FIELDS 2>&1 - - -#----------------------------------------------------------------------------- -# clean-up - -rm -f ncstorage.tmp* -rm -f nml.log NAMELIST_SUB NAMELIST_ICONREMAP NAMELIST_ICONREMAP_FIELDS - -#----------------------------------------------------------------------------- -exit -#----------------------------------------------------------------------------- diff --git a/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg b/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg deleted file mode 100755 index 38dc6b07..00000000 --- a/cases/icon-seq-test/icontools_remap_lbc_rest_runjob.cfg +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -#SBATCH --job-name=iconremap_lbc -#SBATCH --account={cfg.compute_account} -#SBATCH --chdir={cfg.icon_work} -#SBATCH --partition={cfg.compute_queue} -#SBATCH --constraint={cfg.constraint} -#SBATCH --nodes=1 -#SBATCH --ntasks-per-core=1 -#SBATCH --ntasks-per-node={cfg.ntasks_per_node} -#SBATCH --cpus-per-task=1 -#SBATCH --output={logfile} -#SBATCH --open-mode=append - -ulimit -s unlimited - -set -x - -export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions - -. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh -spack load icontools - -#----------------------------------------------------------------------------- -# Extract boundary data -#----------------------------------------------------------------------------- - -rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* - -cat > NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} << EOF -! -&input_field_nml ! temperature - inputname = "T" - outputname = "T" - code = 130 - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. u - inputname = "U" - outputname = "U" - intp_method = 3 -/ -&input_field_nml ! horiz. wind comp. v - inputname = "V" - outputname = "V" - intp_method = 3 -/ -&input_field_nml ! vertical velocity - inputname = "OMEGA" - outputname = "W" - code = 135 - intp_method = 3 -/ -&input_field_nml ! surface pressure - inputname = "LNSP" - outputname = "LNPS" - code = 152 - intp_method = 3 -/ -&input_field_nml ! specific humidity - inputname = "QV" - outputname = "QV" - code = 133 - intp_method = 3 -/ -&input_field_nml ! cloud liquid water content - inputname = "CLWC" - outputname = "QC" - code = 246 - intp_method = 3 -/ -&input_field_nml ! cloud ice water content - inputname = "CIWC" - outputname = "QI" - code = 247 - intp_method = 3 -/ -&input_field_nml ! rain water content - inputname = "CRWC" - outputname = "QR" - code = 75 - intp_method = 3 -/ -&input_field_nml ! snow water content - inputname = "CSWC" - outputname = "QS" - code = 76 - intp_method = 3 -/ -EOF - -#----------------------------------------------------------------------------- -# loop over file list: - -echo "DATAFILELIST is {datafile_list_rest}" -for datafilename in {datafile_list_rest} ; do - datafile="${{datafilename##*/}}" # get filename without path - outdatafile=${{datafile%.*}} # get filename without suffix - cat > NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} << EOF_2C -&remap_nml - in_grid_filename = '{cfg.meteo_dir}/{cfg.meteo_prefix}{cfg.startdate_sim_yyyymmddhh}{cfg.meteo_suffix}' - in_filename = '{cfg.meteo_dir}/${{datafile}}' - in_type = 1 - out_grid_filename = '{cfg.input_files_scratch_lateral_boundary_grid}' - out_filename = '{cfg.icon_input_icbc}/${{outdatafile}}_lbc.nc' - out_type = 2 - out_filetype = 4 - l_have3dbuffer = .false. -! ncstorage_file = "ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}" -/ -EOF_2C - - srun -n 1 {cfg.iconremap_bin} -q \ - --remap_nml NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} \ - --input_field_nml NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} 2>&1 - -done - -#----------------------------------------------------------------------------- -# clean-up - -#rm -f ncstorage.tmp_lbc_{cfg.startdate_sim_yyyymmddhh}* -rm -f nml.log -rm -f NAMELIST_ICONSUB_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_lbc_{cfg.startdate_sim_yyyymmddhh} NAMELIST_ICONREMAP_FIELDS_{cfg.startdate_sim_yyyymmddhh} - -#----------------------------------------------------------------------------- -exit -#----------------------------------------------------------------------------- - From ee98de11d7caaa803ca492e23fc89e0ea39de39d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 16:14:16 +0100 Subject: [PATCH 239/245] Add dependencies to all workflows --- workflows.yaml | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/workflows.yaml b/workflows.yaml index c5c574c0..f87c22b5 100644 --- a/workflows.yaml +++ b/workflows.yaml @@ -1,18 +1,26 @@ cosmo: features: - restart - - spinup jobs: - prepare_cosmo - int2lm - cosmo - post_cosmo + dependencies: + int2lm: + current: + - prepare_cosmo + cosmo: + previous: + - cosmo + post_cosmo: + current: + - cosmo cosmo-ghg: features: - restart - tracers - - spinup jobs: - prepare_cosmo - emissions @@ -116,6 +124,28 @@ cosmo-art: - int2lm - cosmo - post_cosmo + dependencies: + emissions: + current: + - prepare_cosmo + obs_nudging: + current: + - prepare_cosmo + photo_rate: + current: + - prepare_cosmo + int2lm: + current: + - prepare_cosmo + - emissions + - obs_nudging + - photo_rate + cosmo: + previous: + - cosmo + post_cosmo: + current: + - cosmo icon: features: From f4533bbb78e0572d8fe77a2be4dc609d03ee87d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20J=C3=A4hn?= Date: Wed, 31 Jan 2024 16:25:22 +0100 Subject: [PATCH 240/245] Remove is_async config variable --- config.py | 10 +++++----- jobs/post_cosmo.py | 4 ---- run_chain.py | 1 - 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/config.py b/config.py index 3827f974..188893dc 100644 --- a/config.py +++ b/config.py @@ -196,7 +196,7 @@ def set_workflow(self): "Invalid workflow type. Must be either a string or a dictionary." ) - self.is_async = 'dependencies' in self.workflow + assert 'dependencies' in self.workflow, "Missing 'dependencies' in workflow. Exiting." # Initiate empty job ids dictionnary so that it can be filled in later self.job_ids = {'current': {}, 'previous': {}} @@ -381,7 +381,7 @@ def get_dep_ids(self, job_name, add_dep=None): dep_id_list = [] # Add job dependencies - if self.is_async: + if not self.force_sync: # Could be that job has no dependency, even in an async config, # e.g., prepare_data if deps := self.workflow['dependencies'].get(job_name): @@ -396,8 +396,8 @@ def get_dep_ids(self, job_name, add_dep=None): def get_dep_cmd(self, job_name, add_dep=None): """Generate the part of the sbatch command that sepcifies dependencies for job_name.""" - if self.is_async: - # async case + if not self.force_sync: + # Default: async case if dep_ids := self.get_dep_ids(job_name, add_dep=add_dep): dep_str = ':'.join(map(str, dep_ids)) return f'--dependency=afterok:{dep_str}' @@ -406,7 +406,7 @@ def get_dep_cmd(self, job_name, add_dep=None): # so don't use --wait return None else: - # sequential case + # Needed for nested run_chain.py return '--wait' def submit(self, job_name, script, add_dep=None): diff --git a/jobs/post_cosmo.py b/jobs/post_cosmo.py index 22df632d..0442915a 100644 --- a/jobs/post_cosmo.py +++ b/jobs/post_cosmo.py @@ -113,10 +113,6 @@ def main(cfg): logs_src=str(cfg.log_finished_dir).rstrip('/'), logs_dest=logs_path.rstrip('/')) - if not cfg.is_async: - # Wait for Cosmo to finish first - tools.check_job_completion(cfg.log_finished_dir, "cosmo") - os.makedirs(cfg.cosmo_run, exist_ok=True) script = (cfg.cosmo_run / 'run_post_cosmo.job') with open(script, "w") as outf: diff --git a/run_chain.py b/run_chain.py index 33cb4cf7..02b49950 100755 --- a/run_chain.py +++ b/run_chain.py @@ -329,7 +329,6 @@ def main(): # Check sync is forced if args.force_sync: - cfg.is_async = None cfg.force_sync = True else: cfg.force_sync = False From c5e545750f3289aa274ee7351811737b9851715a Mon Sep 17 00:00:00 2001 From: efmkoene Date: Mon, 5 Feb 2024 11:42:32 +0100 Subject: [PATCH 241/245] Include renaming scheme for chemicals for OEM --- cases/icon-art-oem-test/config.yaml | 2 ++ jobs/prepare_art_oem.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index 475d326a..6675c273 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -41,6 +41,8 @@ chem: nameformat: '%Y%m%d_%H' suffix: .grb inc: 3 + chem_in_to_chem_out_remap_tracers: + TRCH4_chemtr: CH4_BG icontools_runjobs: - icontools_remap_ic_runjob.cfg diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 8e4abe17..c5822c35 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -74,7 +74,10 @@ def main(cfg): ds_chem['PS'] = ds_chem['LNPS'] ds_chem['PS'].attrs = ds_chem['LNPS'].attrs ds_chem['PS'].attrs["long_name"] = 'surface pressure' - ds_chem['TRCH4_chemtr'] = ds_chem['CH4_BG'] + # Remapping chemical tracer names + if "chem_in_to_chem_out_remap_tracers" in cfg.chem: + for chem_in, chem_out in cfg.chem['chem_in_to_chem_out_remap_tracers'].items(): + ds_chem[chem_out] = ds_chem[chem_in] # merge: ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") #ds_merged.attrs = ds.attrs From 6ede4bba256d9d7f4ed857b5c33b2e098d5d14e6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 5 Feb 2024 10:42:54 +0000 Subject: [PATCH 242/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_art_oem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index c5822c35..ae79f916 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -76,7 +76,8 @@ def main(cfg): ds_chem['PS'].attrs["long_name"] = 'surface pressure' # Remapping chemical tracer names if "chem_in_to_chem_out_remap_tracers" in cfg.chem: - for chem_in, chem_out in cfg.chem['chem_in_to_chem_out_remap_tracers'].items(): + for chem_in, chem_out in cfg.chem[ + 'chem_in_to_chem_out_remap_tracers'].items(): ds_chem[chem_out] = ds_chem[chem_in] # merge: ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") From 21400496f5f823ad3ef6e6a77efcdf6e06339fd0 Mon Sep 17 00:00:00 2001 From: efmkoene Date: Mon, 5 Feb 2024 12:31:29 +0100 Subject: [PATCH 243/245] re-order in and output tracers --- cases/icon-art-oem-test/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index 6675c273..3f60d797 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -42,7 +42,7 @@ chem: suffix: .grb inc: 3 chem_in_to_chem_out_remap_tracers: - TRCH4_chemtr: CH4_BG + CH4_BG: TRCH4_chemtr icontools_runjobs: - icontools_remap_ic_runjob.cfg From 0325f66f72ab701ec272b3e0a107eda82835f437 Mon Sep 17 00:00:00 2001 From: efmkoene Date: Mon, 5 Feb 2024 15:41:46 +0100 Subject: [PATCH 244/245] Shortened variable name --- cases/icon-art-oem-test/config.yaml | 2 +- jobs/prepare_art_oem.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cases/icon-art-oem-test/config.yaml b/cases/icon-art-oem-test/config.yaml index 3f60d797..a50fbacc 100644 --- a/cases/icon-art-oem-test/config.yaml +++ b/cases/icon-art-oem-test/config.yaml @@ -41,7 +41,7 @@ chem: nameformat: '%Y%m%d_%H' suffix: .grb inc: 3 - chem_in_to_chem_out_remap_tracers: + remap_tracers: CH4_BG: TRCH4_chemtr icontools_runjobs: diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index ae79f916..7990c9d9 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -75,9 +75,9 @@ def main(cfg): ds_chem['PS'].attrs = ds_chem['LNPS'].attrs ds_chem['PS'].attrs["long_name"] = 'surface pressure' # Remapping chemical tracer names - if "chem_in_to_chem_out_remap_tracers" in cfg.chem: + if "remap_tracers" in cfg.chem: for chem_in, chem_out in cfg.chem[ - 'chem_in_to_chem_out_remap_tracers'].items(): + 'remap_tracers'].items(): ds_chem[chem_out] = ds_chem[chem_in] # merge: ds_merged = xr.merge([ds_meteo, ds_chem], compat="override") From 483193bda4447fd815fef873ea8328e5d9a8595e Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 5 Feb 2024 14:42:10 +0000 Subject: [PATCH 245/245] GitHub Action: Apply Pep8-formatting --- jobs/prepare_art_oem.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/prepare_art_oem.py b/jobs/prepare_art_oem.py index 7990c9d9..b421aa1d 100644 --- a/jobs/prepare_art_oem.py +++ b/jobs/prepare_art_oem.py @@ -76,8 +76,7 @@ def main(cfg): ds_chem['PS'].attrs["long_name"] = 'surface pressure' # Remapping chemical tracer names if "remap_tracers" in cfg.chem: - for chem_in, chem_out in cfg.chem[ - 'remap_tracers'].items(): + for chem_in, chem_out in cfg.chem['remap_tracers'].items(): ds_chem[chem_out] = ds_chem[chem_in] # merge: ds_merged = xr.merge([ds_meteo, ds_chem], compat="override")