Run ert style on Everest

.pre-commit-config.yaml

@@ -14,5 +14,5 @@ repos:
     rev: v0.6.5
     hooks:
     - id: ruff
-      args: [ --fix ]
+      args: [ --fix]
     - id: ruff-format

src/everest/__init__.py

@@ -25,21 +25,21 @@
 
 __author__ = "Equinor ASA and TNO"
 __all__ = [
-    "load",
-    "start_optimization",
+    "SIMULATOR_END",
     "SIMULATOR_START",
     "SIMULATOR_UPDATE",
-    "SIMULATOR_END",
     "ConfigKeys",
-    "export",
-    "export_with_progress",
-    "export_to_csv",
-    "validate_export",
-    "filter_data",
     "MetaDataColumnNames",
     "detached",
     "docs",
+    "export",
+    "export_to_csv",
+    "export_with_progress",
+    "filter_data",
     "jobs",
+    "load",
+    "start_optimization",
     "templates",
     "util",
+    "validate_export",
 ]

src/everest/api/everest_data_api.py

@@ -1,9 +1,9 @@
 from collections import OrderedDict
 
 import pandas as pd
-from ert.storage import open_storage
 from seba_sqlite.snapshot import SebaSnapshot
 
+from ert.storage import open_storage
 from everest.config import EverestConfig
 from everest.detached import ServerStatus, everserver_status
 

src/everest/bin/everest_script.py

@@ -9,7 +9,6 @@
 
 from ert.config import ErtConfig
 from ert.storage import open_storage
-
 from everest.config import EverestConfig
 from everest.detached import (
     ServerStatus,

src/everest/bin/everload_script.py

@@ -10,7 +10,6 @@
 from ert import LibresFacade
 from ert.config import ErtConfig
 from ert.storage import open_storage
-
 from everest import MetaDataColumnNames as MDCN
 from everest import export
 from everest.config import EverestConfig
@@ -82,7 +81,7 @@ def batch(batch_str, parser=arg_parser):
         batch_str = "{}".format(
             batch_str.strip()
         )  # Because isnumeric only works on unicode strings in py27
-        if not batch_str.isnumeric() or batch_str[0] == "0" and len(batch_str) > 1:
+        if not batch_str.isnumeric() or (batch_str[0] == "0" and len(batch_str) > 1):
             parser.error("Invalid batch given: '{}'".format(batch_str))
         return int(batch_str)
 

src/everest/bin/kill_script.py

@@ -85,7 +85,7 @@ def kill_everest(options):
         print("Waiting for server to stop ...")
         wait_for_server_to_stop(options.config, timeout=60)
         print("Server stopped.")
-    except:  # noqa E722
+    except:
         logging.debug(traceback.format_exc())
         print(
             "Server is still running after 60 seconds, "

src/everest/bin/main.py

@@ -3,8 +3,6 @@
 import logging
 import sys
 
-from ieverest.bin.ieverest_script import ieverest_entry
-
 from everest import __version__ as everest_version
 from everest import docs
 from everest.bin.config_branch_script import config_branch_entry
@@ -17,6 +15,7 @@
 from everest.bin.monitor_script import monitor_entry
 from everest.bin.visualization_script import visualization_entry
 from everest.util import configure_logger
+from ieverest.bin.ieverest_script import ieverest_entry
 
 
 def _create_dump_action(dumps, extended=False):

src/everest/bin/utils.py

@@ -4,12 +4,12 @@
 import traceback
 from dataclasses import dataclass, field
 from itertools import groupby
-from typing import Dict, List
+from typing import ClassVar, Dict, List
 
 import colorama
 from colorama import Fore
-from ert.simulator.batch_simulator_context import Status
 
+from ert.simulator.batch_simulator_context import Status
 from everest.config import EverestConfig
 from everest.detached import (
     OPT_PROGRESS_ID,
@@ -108,7 +108,7 @@ class JobProgress:
             JOB_FAILURE: [],  # contains failed simulation numbers i.e [5,6]
         }
     )
-    STATUS_COLOR = {
+    STATUS_COLOR: ClassVar = {
         JOB_RUNNING: Fore.BLUE,
         JOB_SUCCESS: Fore.GREEN,
         JOB_FAILURE: Fore.RED,
@@ -173,7 +173,7 @@ def update(self, status):
                     print(msg)
                     self._clear_lines = len(msg.split("\n"))
                     self._last_reported_batch = batch
-        except:  # noqa E722
+        except:
             logging.getLogger(EVEREST).debug(traceback.format_exc())
 
     def get_opt_progress(self, context_status):

src/everest/config/__init__.py

@@ -23,12 +23,12 @@
 from .workflow_config import WorkflowConfig
 
 __all__ = [
-    "EverestConfig",
+    "CVaRConfig",
     "ControlConfig",
     "ControlVariableConfig",
     "ControlVariableGuessListConfig",
-    "CVaRConfig",
     "EnvironmentConfig",
+    "EverestConfig",
     "ExportConfig",
     "InputConstraintConfig",
     "InstallDataConfig",

src/everest/config/everest_config.py

@@ -7,7 +7,6 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Literal, Optional, Protocol, no_type_check
 
-from ert.config import ErtConfig
 from pydantic import (
     AfterValidator,
     BaseModel,
@@ -20,6 +19,7 @@
 from ruamel.yaml import YAML, YAMLError
 from typing_extensions import Annotated
 
+from ert.config import ErtConfig
 from everest.config.control_variable_config import ControlVariableGuessListConfig
 from everest.config.install_template_config import InstallTemplateConfig
 from everest.config.server_config import ServerConfig
@@ -324,7 +324,7 @@ def validate_workflow_name_installed(self):  # pylint: disable=E0213
 
     @field_validator("install_templates")
     @classmethod
-    def validate_install_templates_unique_output_files(cls, install_templates):  # pylint: disable=E0213 # noqa: E501
+    def validate_install_templates_unique_output_files(cls, install_templates):  # pylint: disable=E0213
         check_for_duplicate_names(
             [t.output_file for t in install_templates],
             "install_templates",
@@ -791,7 +791,7 @@ def load_file_with_argparser(
         except YAMLError as e:
             parser.error(
                 f"The config file: <{config_path}> contains"
-                f" invalid YAML syntax: {str(e)}"
+                f" invalid YAML syntax: {e!s}"
             )
         except ValidationError as e:
             parser.error(

src/everest/config/model_config.py

@@ -33,7 +33,7 @@ class ModelConfig(BaseModel, extra="forbid"):  # type: ignore
 
     @model_validator(mode="before")
     @classmethod
-    def validate_realizations_weights_same_cardinaltiy(cls, values):  # pylint: disable=E0213 # noqa: E501
+    def validate_realizations_weights_same_cardinaltiy(cls, values):  # pylint: disable=E0213
         weights = values.get("realizations_weights")
         reals = values.get("realizations")
 

src/everest/config_file_loader.py

@@ -95,7 +95,7 @@ class Os(object):
 
 def _render_definitions(definitions, jinja_env):
     # pylint: disable=unnecessary-lambda-assignment
-    render = lambda s, d: jinja_env.from_string(s).render(**d)  # noqa
+    render = lambda s, d: jinja_env.from_string(s).render(**d)
     for key in definitions:
         if not isinstance(definitions[key], str):
             continue

src/everest/detached/__init__.py

@@ -10,11 +10,11 @@
 
 import pkg_resources
 import requests
-from ert import BatchContext, BatchSimulator
-from ert.config import ErtConfig, QueueSystem
 from seba_sqlite.exceptions import ObjectNotFoundError
 from seba_sqlite.snapshot import SebaSnapshot
 
+from ert import BatchContext, BatchSimulator
+from ert.config import ErtConfig, QueueSystem
 from everest.config import EverestConfig
 from everest.config_keys import ConfigKeys as CK
 from everest.simulator import JOB_FAILURE, JOB_SUCCESS, Status
@@ -136,7 +136,7 @@ def stop_server(config: EverestConfig, retries: int = 5):
             )
             response.raise_for_status()
             return True
-        except:  # noqa
+        except:
             logging.debug(traceback.format_exc())
             time.sleep(retry)
     return False
@@ -309,7 +309,7 @@ def server_is_running(config: EverestConfig):
             proxies=PROXY,  # type: ignore
         )
         response.raise_for_status()
-    except:  # noqa
+    except:
         logging.debug(traceback.format_exc())
         return False
     return True
@@ -356,7 +356,7 @@ def start_monitor(config: EverestConfig, callback, polling_interval=5):
                 ret = bool(callback({OPT_PROGRESS_ID: opt_status}))
                 stop |= ret
             time.sleep(polling_interval)
-    except:  # noqa
+    except:
         logging.debug(traceback.format_exc())
 
 
@@ -490,7 +490,7 @@ def generate_everserver_ert_config(config: EverestConfig, debug_mode: bool = Fal
     everserver_config["INSTALL_JOB"] = install_job
 
     simulation_job = everserver_config.get("SIMULATION_JOB", [])
-    simulation_job.append([EVEREST_SERVER_CONFIG] + arg_list)
+    simulation_job.append([EVEREST_SERVER_CONFIG, *arg_list])
     everserver_config["SIMULATION_JOB"] = simulation_job
 
     if queue_system in _QUEUE_SYSTEMS:

src/everest/detached/jobs/everserver.py

@@ -260,7 +260,7 @@ def main():
         )
         everserver_instance.daemon = True
         everserver_instance.start()
-    except:  # noqa
+    except:
         update_everserver_status(
             config, ServerStatus.failed, message=traceback.format_exc()
         )
@@ -277,7 +277,7 @@ def main():
         if status != ServerStatus.completed:
             update_everserver_status(config, status, message)
             return
-    except:  # noqa
+    except:
         if shared_data[STOP_ENDPOINT]:
             update_everserver_status(
                 config, ServerStatus.stopped, message="Optimization aborted."
@@ -295,7 +295,7 @@ def main():
         for msg in err_msgs:
             logging.getLogger(EVEREST).warning(msg)
         export_to_csv(config, export_ecl=export_ecl)
-    except:  # noqa
+    except:
         update_everserver_status(
             config, ServerStatus.failed, message=traceback.format_exc()
         )

src/everest/export.py

@@ -4,10 +4,10 @@
 from typing import Any, Dict, List, Optional, Set
 
 import pandas as pd
-from ert.storage import open_storage
 from pandas import DataFrame
 from seba_sqlite.snapshot import SebaSnapshot
 
+from ert.storage import open_storage
 from everest.config import EverestConfig
 from everest.strings import STORAGE_DIR
 
@@ -134,7 +134,7 @@ def _metadata(config: EverestConfig):
                 opt = opt_data[md_row[MetaDataColumnNames.BATCH]]
                 md_row.update(
                     {
-                        MetaDataColumnNames.REAL_AVERAGED_OBJECTIVE: opt.objective_value,  # noqa
+                        MetaDataColumnNames.REAL_AVERAGED_OBJECTIVE: opt.objective_value,
                         MetaDataColumnNames.INCREASED_MERIT: opt.merit_flag,
                     }
                 )

src/everest/jobs/__init__.py

@@ -18,11 +18,11 @@
 )
 
 __all__ = [
-    "recovery_factor",
     "io",
+    "recovery_factor",
+    "script_names",
     "templating",
     "well_tools",
-    "script_names",
 ]
 
 

src/everest/jobs/well_tools/__init__.py

@@ -51,11 +51,11 @@ def well_set(well_data_file, new_entry_file, output_file):
         err_msg = "Expected there to be exactly one new entry " "in {nef}, was {ne}"
         raise ValueError(err_msg.format(nef=new_entry_file, ne=len(new_entry)))
 
-    entry_key = list(new_entry.keys())[0]
+    entry_key = next(iter(new_entry.keys()))
     entry_data = new_entry[entry_key]
 
     if len(well_data) != len(entry_data):
-        err_msg = "Expected number of entries in {well_data_file} to be equal "
+        err_msg = f"Expected number of entries in {well_data_file} to be equal "
         "the number of entries in {new_entry_file} ({nwell} != {nentry})"
         err_msg = err_msg.format(
             well_data_file=well_data_file,
@@ -121,7 +121,7 @@ def well_opdate_filter(well_data_file, start_date, end_date, output_file):
     well_data = everest.jobs.io.load_data(well_data_file)
 
     # pylint: disable=unnecessary-lambda-assignment
-    valid = lambda well: _valid_operational_dates(well, start_date, end_date)  # noqa
+    valid = lambda well: _valid_operational_dates(well, start_date, end_date)
     well_data = list(filter(valid, well_data))
 
     with everest.jobs.io.safe_open(output_file, "w") as fout:

src/everest/plugins/plugin_response.py

@@ -2,7 +2,7 @@
 
 
 @decorator
-def plugin_response(func, plugin_name="", *args, **kwargs):  # pylint: disable=keyword-arg-before-vararg # noqa: E501
+def plugin_response(func, plugin_name="", *args, **kwargs):  # pylint: disable=keyword-arg-before-vararg
     response = func(*args, **kwargs)
     return (
         PluginResponse(response, PluginMetadata(plugin_name, func.__name__))

src/everest/plugins/site_config_env.py

@@ -27,7 +27,7 @@ def _config_env_vars(self):
             if env_value is not None
         ]
 
-        return config_lines + [""]
+        return [*config_lines, ""]
 
     def _get_temp_site_config_path(self):
         self.tmp_dir = tempfile.mkdtemp()

src/everest/queue_driver/queue_driver.py

@@ -1,7 +1,6 @@
 from typing import Any, List, Optional, Tuple
 
 from ert.config import QueueSystem
-
 from everest.config import EverestConfig
 from everest.config.simulator_config import SimulatorConfig
 from everest.config_keys import ConfigKeys

src/everest/simulator/__init__.py

@@ -1,5 +1,4 @@
 from ert.simulator.batch_simulator_context import Status
-
 from everest.simulator.simulator import Simulator
 
 JOB_SUCCESS = "Success"
@@ -107,10 +106,10 @@
 )
 
 __all__ = [
-    "Simulator",
-    "Status",
+    "JOB_FAILURE",
+    "JOB_RUNNING",
     "JOB_SUCCESS",
     "JOB_WAITING",
-    "JOB_RUNNING",
-    "JOB_FAILURE",
+    "Simulator",
+    "Status",
 ]

src/everest/simulator/everest_to_ert.py

@@ -398,9 +398,9 @@ def _extract_templating(ever_config: EverestConfig):
                 tmpl_request.output_file,
                 "--template",
                 tmpl_request.template,
+                "--input_files",
+                *res_input,
             ]
-            + ["--input_files"]
-            + res_input
         )
         forward_model.append(f"render {args}")