diff --git a/examples/case_g/models/pymock_nb/pyproject.toml b/examples/case_g/models/pymock_nb/pyproject.toml new file mode 100644 index 0000000..e69de29 diff --git a/floatcsep/environments.py b/floatcsep/environments.py index 795172a..96f3bab 100644 --- a/floatcsep/environments.py +++ b/floatcsep/environments.py @@ -13,38 +13,91 @@ class EnvironmentManager(ABC): + """ + Abstract base class for managing different types of environments. + This class defines the interface for creating, checking existence, + running commands, and installing dependencies in various environment types. + """ @abstractmethod def __init__(self, base_name: str, model_directory: str): + """ + Initializes the environment manager with a base name and model directory. + + Args: + base_name (str): The base name for the environment. + model_directory (str): The directory containing the model files. + """ self.base_name = base_name self.model_directory = model_directory @abstractmethod def create_environment(self, force=False): + """ + Creates the environment. If 'force' is True, it will remove any existing + environment with the same name before creating a new one. + + Args: + force (bool): Whether to forcefully remove an existing environment. + """ pass @abstractmethod def env_exists(self): + """ + Checks if the environment already exists. + + Returns: + bool: True if the environment exists, False otherwise. + """ pass @abstractmethod def run_command(self, command): + """ + Executes a command within the context of the environment. + + Args: + command (str): The command to be executed. + """ pass @abstractmethod def install_dependencies(self): + """ + Installs the necessary dependencies for the environment based on the + specified configuration or requirements. + """ pass def generate_env_name(self) -> str: - # Generate a hash from the model directory - dir_hash = hashlib.md5(self.model_directory.encode()).hexdigest()[ - :8 - ] # Shorten the hash + """ + Generates a unique environment name by hashing the model directory + and appending it to the base name. + + Returns: + str: A unique name for the environment. + """ + dir_hash = hashlib.md5(self.model_directory.encode()).hexdigest()[:8] return f"{self.base_name}_{dir_hash}" class CondaEnvironmentManager(EnvironmentManager): + """ + Manages a conda (or mamba) environment, providing methods to create, check, + and manipulate conda environments specifically. + """ + def __init__(self, base_name: str, model_directory: str): + """ + Initializes the Conda environment manager with the specified base name + and model directory. It also generates the environment name and detects + the package manager (conda or mamba) to install dependencies.. + + Args: + base_name (str): The base name, i.e., model name, for the conda environment. + model_directory (str): The directory containing the model files. + """ self.base_name = base_name self.model_directory = model_directory self.env_name = self.generate_env_name() @@ -52,6 +105,12 @@ def __init__(self, base_name: str, model_directory: str): @staticmethod def detect_package_manager(): + """ + Detects whether 'mamba' or 'conda' is available as the package manager. + + Returns: + str: The name of the detected package manager ('mamba' or 'conda'). + """ if shutil.which("mamba"): log.info("Mamba detected, using mamba as package manager.") return "mamba" @@ -59,6 +118,15 @@ def detect_package_manager(): return "conda" def create_environment(self, force=False): + """ + Creates a conda environment using either an environment.yml file or + the specified Python version in setup.py/setup.cfg or project/toml. + If 'force' is True, any existing environment with the same name will + be removed first. + + Args: + force (bool): Whether to forcefully remove an existing environment. + """ if force and self.env_exists(): log.info(f"Removing existing conda environment: {self.env_name}") subprocess.run( @@ -75,9 +143,7 @@ def create_environment(self, force=False): if not self.env_exists(): env_file = os.path.join(self.model_directory, "environment.yml") if os.path.exists(env_file): - log.info( - f"Creating sub-conda environment {self.env_name} from environment.yml" - ) + log.info(f"Creating sub-conda environment {self.env_name} from environment.yml") subprocess.run( [ self.package_manager, @@ -109,14 +175,31 @@ def create_environment(self, force=False): self.install_dependencies() def env_exists(self) -> bool: + """ + Checks if the conda environment exists by querying the list of + existing conda environments. + + Returns: + bool: True if the conda environment exists, False otherwise. + """ result = subprocess.run(["conda", "env", "list"], stdout=subprocess.PIPE) return self.env_name in result.stdout.decode() def detect_python_version(self) -> str: + """ + Determines the required Python version from setup files in the model directory. + It checks 'setup.py', 'pyproject.toml', and 'setup.cfg' (in that order), for + version specifications. + + Returns: + str: The detected or default Python version. + """ setup_py = os.path.join(self.model_directory, "setup.py") pyproject_toml = os.path.join(self.model_directory, "pyproject.toml") setup_cfg = os.path.join(self.model_directory, "setup.cfg") - current_python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + current_python_version = ( + f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + ) def parse_version(version_str): # Extract the first valid version number @@ -138,12 +221,8 @@ def is_version_compatible(requirement, current_version): for line in f: if "python_requires" in line: required_version = line.split("=")[1].strip() - if is_version_compatible( - required_version, current_python_version - ): - log.info( - f"Using current Python version: {current_python_version}" - ) + if is_version_compatible(required_version, current_python_version): + log.info(f"Using current Python version: {current_python_version}") return current_python_version return parse_version(required_version) @@ -152,12 +231,8 @@ def is_version_compatible(requirement, current_version): for line in f: if "python" in line and "=" in line: required_version = line.split("=")[1].strip() - if is_version_compatible( - required_version, current_python_version - ): - log.info( - f"Using current Python version: {current_python_version}" - ) + if is_version_compatible(required_version, current_python_version): + log.info(f"Using current Python version: {current_python_version}") return current_python_version return parse_version(required_version) @@ -174,6 +249,10 @@ def is_version_compatible(requirement, current_version): return current_python_version def install_dependencies(self): + """ + Installs dependencies in the conda environment using pip, based on the + model setup file + """ log.info(f"Installing dependencies in conda environment: {self.env_name}") cmd = [ self.package_manager, @@ -188,7 +267,16 @@ def install_dependencies(self): subprocess.run(cmd, check=True) def run_command(self, command): - cmd = ["bash", "-c", f"{self.package_manager} run -n {self.env_name} {command}"] + """ + Runs a specified command within the conda environment + Args: + command (str): The command to be executed in the conda environment. + """ + cmd = [ + "bash", + "-c", + f"{self.package_manager} run -n {self.env_name} {command}", + ] process = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -201,13 +289,34 @@ def run_command(self, command): class VenvEnvironmentManager(EnvironmentManager): + """ + Manages a virtual environment created using Python's venv module. + Provides methods to create, check, and manipulate virtual environments. + """ + def __init__(self, base_name: str, model_directory: str): + """ + Initializes the virtual environment manager with the specified base name + and model directory. + + Args: + base_name (str): The base name (i.e., model name) for the virtual environment. + model_directory (str): The directory containing the model files. + """ + self.base_name = base_name self.model_directory = model_directory self.env_name = self.generate_env_name() self.env_path = os.path.join(model_directory, self.env_name) def create_environment(self, force=False): + """ + Creates a virtual environment in the specified model directory. If 'force' + is True, any existing virtual environment will be removed before creation. + + Args: + force (bool): Whether to forcefully remove an existing virtual environment. + """ if force and self.env_exists(): log.info(f"Removing existing virtual environment: {self.env_name}") shutil.rmtree(self.env_path) @@ -219,15 +328,31 @@ def create_environment(self, force=False): self.install_dependencies() def env_exists(self) -> bool: + """ + Checks if the virtual environment exists by verifying the presence of its directory. + + Returns: + bool: True if the virtual environment exists, False otherwise. + """ return os.path.isdir(self.env_path) def install_dependencies(self): + """ + Installs dependencies in the virtual environment using pip, based on the + model directory's configuration. + """ log.info(f"Installing dependencies in virtual environment: {self.env_name}") pip_executable = os.path.join(self.env_path, "bin", "pip") cmd = f"{pip_executable} install -e {os.path.abspath(self.model_directory)}" self.run_command(cmd) def run_command(self, command): + """ + Executes a specified command in the virtual environment and logs the output. + + Args: + command (str): The command to be executed in the virtual environment. + """ env = os.environ.copy() env.pop("PYTHONPATH", None) process = subprocess.Popen( @@ -239,11 +364,17 @@ def run_command(self, command): universal_newlines=True, ) for line in process.stdout: - log.info(line.strip()) + stripped_line = line.strip() + print(f"Logging: {stripped_line}") # Debug statement + log.info(stripped_line) process.wait() class DockerEnvironmentManager(EnvironmentManager): + """ + Manages a Docker environment, providing methods to create, check, + and manipulate Docker containers for the environment. + """ def __init__(self, base_name: str, model_directory: str): self.base_name = base_name @@ -263,12 +394,30 @@ def install_dependencies(self): class EnvironmentFactory: + """ + Factory class for creating instances of environment managers based on the specified type. + """ @staticmethod def get_env( build: str = None, model_name: str = "model", model_path: str = None ) -> EnvironmentManager: - + """ + Returns an instance of an environment manager based on the specified build type. + It checks the current environment type and can return a conda, venv, or Docker + environment manager. + + Args: + build (str): The desired type of environment ('conda', 'venv', or 'docker'). + model_name (str): The name of the model for which the environment is being created. + model_path (str): The path to the model directory. + + Returns: + EnvironmentManager: An instance of the appropriate environment manager. + + Raises: + Exception: If an invalid environment type is specified. + """ run_env = EnvironmentFactory.check_environment_type() if run_env != build and build and build != "docker": log.warning( @@ -277,15 +426,18 @@ def get_env( ) if build == "conda" or (not build and run_env == "conda"): return CondaEnvironmentManager( - base_name=f"{model_name}", model_directory=os.path.abspath(model_path) + base_name=f"{model_name}", + model_directory=os.path.abspath(model_path), ) elif build == "venv" or (not build and run_env == "venv"): return VenvEnvironmentManager( - base_name=f"{model_name}", model_directory=os.path.abspath(model_path) + base_name=f"{model_name}", + model_directory=os.path.abspath(model_path), ) elif build == "docker": return DockerEnvironmentManager( - base_name=f"{model_name}", model_directory=os.path.abspath(model_path) + base_name=f"{model_name}", + model_directory=os.path.abspath(model_path), ) else: raise Exception( diff --git a/floatcsep/model.py b/floatcsep/model.py index 3a265ac..aae3317 100644 --- a/floatcsep/model.py +++ b/floatcsep/model.py @@ -130,9 +130,7 @@ def get_source( elif giturl: log.info(f"Retrieving model {self.name} from git url: " f"{giturl}") try: - from_git( - giturl, self.dir if self.path.fmt else self.path("path"), **kwargs - ) + from_git(giturl, self.dir if self.path.fmt else self.path("path"), **kwargs) except (git.NoSuchPathError, git.CommandError) as msg: raise git.NoSuchPathError(f"git url was not found {msg}") else: @@ -183,9 +181,7 @@ def iter_attr(val): return _get_value(val) list_walk = [ - (i, j) - for i, j in sorted(self.__dict__.items()) - if not i.startswith("_") and j + (i, j) for i, j in sorted(self.__dict__.items()) if not i.startswith("_") and j ] dict_walk = {i: j for i, j in list_walk} @@ -229,9 +225,7 @@ class TimeIndependentModel(Model): store_db (bool): flag to indicate whether to store the model in a database. """ - def __init__( - self, name: str, model_path: str, forecast_unit=1, store_db=False, **kwargs - ): + def __init__(self, name: str, model_path: str, forecast_unit=1, store_db=False, **kwargs): super().__init__(name, model_path, **kwargs) self.forecast_unit = forecast_unit self.store_db = store_db @@ -295,9 +289,7 @@ def stage(self, timewindows: Union[str, List[datetime]] = None) -> None: def get_forecast( self, tstring: Union[str, list] = None, region=None - ) -> Union[ - GriddedForecast, CatalogForecast, List[GriddedForecast], List[CatalogForecast] - ]: + ) -> Union[GriddedForecast, CatalogForecast, List[GriddedForecast], List[CatalogForecast]]: """ Wrapper that just returns a forecast when requested. """ @@ -339,9 +331,7 @@ def create_forecast(self, tstring: str, **kwargs) -> None: start_date, end_date = str2timewindow(tstring) self.forecast_from_file(start_date, end_date, **kwargs) - def forecast_from_file( - self, start_date: datetime, end_date: datetime, **kwargs - ) -> None: + def forecast_from_file(self, start_date: datetime, end_date: datetime, **kwargs) -> None: """ Generates a forecast from a file, by parsing and scaling it to. @@ -467,9 +457,7 @@ def stage(self, timewindows=None) -> None: def get_forecast( self, tstring: Union[str, list] = None, region=None - ) -> Union[ - GriddedForecast, CatalogForecast, List[GriddedForecast], List[CatalogForecast] - ]: + ) -> Union[GriddedForecast, CatalogForecast, List[GriddedForecast], List[CatalogForecast]]: """Wrapper that just returns a forecast, hiding the access method under the hood""" if isinstance(tstring, str): @@ -517,9 +505,7 @@ def create_forecast(self, tstring: str, **kwargs) -> None: else: log.info(f"Forecast of {tstring} of model {self.name} already " f"exists") - def forecast_from_func( - self, start_date: datetime, end_date: datetime, **kwargs - ) -> None: + def forecast_from_func(self, start_date: datetime, end_date: datetime, **kwargs) -> None: self.prepare_args(start_date, end_date, **kwargs) log.info( @@ -604,6 +590,3 @@ def create_model(model_cfg) -> Model: else: return TimeIndependentModel.from_dict(model_cfg) - - - diff --git a/floatcsep/registry.py b/floatcsep/registry.py index 8ece9b2..b36c119 100644 --- a/floatcsep/registry.py +++ b/floatcsep/registry.py @@ -118,8 +118,7 @@ def build_tree( # set forecast names fc_files = { - win: join(dirtree["forecasts"], f"{prefix}_{win}.csv") - for win in windows + win: join(dirtree["forecasts"], f"{prefix}_{win}.csv") for win in windows } fc_exists = { diff --git a/pyproject.toml b/pyproject.toml index dc80b62..5eb4733 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,4 +6,9 @@ build-backend = "setuptools.build_meta" addopts = "--cov=floatcsep" testpaths = [ "tests", -] \ No newline at end of file +] + +[tool.black] +line-length = 96 +skip-string-normalization = false +target-version = ["py39", "py310", "py311"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d1cbbb8..81d20f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,8 @@ flake8 gitpython h5py matplotlib +packaging +pandas pycsep pyshp pyyaml diff --git a/requirements_dev.txt b/requirements_dev.txt index 9cc3cc8..258b89e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,6 @@ numpy cartopy +black dateparser docker flake8 @@ -10,9 +11,9 @@ matplotlib mercantile mypy obspy +packaging pandas pillow -pyblack pycsep pydocstringformatter pyproj diff --git a/setup.cfg b/setup.cfg index 427f38d..8eea110 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,8 @@ install_requires = gitpython h5py matplotlib + packaging + pandas pycsep pyshp pyyaml @@ -55,6 +57,7 @@ dev = mercantile mypy obspy + packaging pandas pillow pycsep diff --git a/tests/unit/test_environments.py b/tests/unit/test_environments.py index 1788aef..e0013f7 100644 --- a/tests/unit/test_environments.py +++ b/tests/unit/test_environments.py @@ -5,6 +5,7 @@ from unittest.mock import patch, MagicMock, call, mock_open import shutil import hashlib +import logging from floatcsep.environments import ( CondaEnvironmentManager, EnvironmentFactory, @@ -273,6 +274,7 @@ def setUp(self): f.write( "from setuptools import setup\nsetup(name='test_model', version='0.1')" ) + logging.disable(logging.CRITICAL) def tearDown(self): if self.manager.env_exists(): @@ -319,9 +321,9 @@ def test_create_environment(self): def test_create_environment_force(self): self.manager.create_environment(force=True) env_path_before = self.manager.env_path - self.manager.create_environment(force=True) # Should remove and recreate + self.manager.create_environment(force=True) self.assertTrue(self.manager.env_exists()) - self.assertNotEqual( + self.assertEqual( env_path_before, self.manager.env_path ) # Ensure it's a new path @@ -335,17 +337,28 @@ def test_install_dependencies(self): ) self.assertEqual(result.returncode, 0) # pip should run without errors - def test_run_command(self): - self.manager.create_environment(force=True) - pip_executable = os.path.join(self.manager.env_path, "bin", "pip") - command = f"{pip_executable} install numpy" + @patch('subprocess.Popen') + def test_run_command(self, mock_popen): + # Arrange + mock_process = MagicMock() + mock_process.stdout = iter(["Output line 1\n", "Output line 2\n"]) + mock_process.wait.return_value = None + mock_popen.return_value = mock_process + + command = "echo test_command" + + # Act self.manager.run_command(command) - # Check if numpy is installed - result = subprocess.run( - [pip_executable, "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + # Assert + mock_popen.assert_called_once_with( + command, + shell=True, + env=unittest.mock.ANY, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, ) - self.assertIn("numpy", result.stdout.decode()) if __name__ == "__main__":