From d25c0961be1329dc546995ec66c65d4ed8fbf0da Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Fri, 15 Nov 2024 17:11:42 +0100 Subject: [PATCH] Resolve import cycles in manage module and its submodules --- .pre-commit-config.yaml | 6 +- docs/source/topics/processes/functions.rst | 4 +- ...y => parse_docstring_expose_ipython.illpy} | 0 ...signature_plain_python_call_illegal.illpy} | 0 src/aiida/cmdline/commands/cmd_profile.py | 8 +- src/aiida/manage/__init__.py | 6 - src/aiida/manage/configuration/__init__.py | 292 +--------------- src/aiida/manage/configuration/config.py | 321 ++++++++---------- .../configuration/migrations/migrations.py | 2 +- src/aiida/manage/configuration/options.py | 12 +- src/aiida/manage/configuration/profile.py | 143 +++++++- .../manage/configuration/schema/__init__.py | 173 ++++++++++ .../tools/pytest_fixtures/configuration.py | 2 +- tests/manage/configuration/test_options.py | 2 +- 14 files changed, 486 insertions(+), 485 deletions(-) rename docs/source/topics/processes/include/snippets/functions/{parse_docstring_expose_ipython.py => parse_docstring_expose_ipython.illpy} (100%) rename docs/source/topics/processes/include/snippets/functions/{signature_plain_python_call_illegal.py => signature_plain_python_call_illegal.illpy} (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d038604e6..3bc4fccf73 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -211,11 +211,7 @@ repos: language: system types: [python] require_serial: true - args: - [ - "--disable=all", - "--enable=cyclic-import", - ] + args: [--disable=all, --enable=cyclic-import] - id: dm-generate-all name: Update all requirements files diff --git a/docs/source/topics/processes/functions.rst b/docs/source/topics/processes/functions.rst index 117a976fa2..0920f47dcc 100644 --- a/docs/source/topics/processes/functions.rst +++ b/docs/source/topics/processes/functions.rst @@ -57,7 +57,7 @@ Notice how the order in which we pass the arguments is irrelevant because we spe Now that we know the difference between positional and named arguments, it is important to realize a python requirement that **positional arguments have to come before named arguments**. What this means is that *both* the function definition and function call below are illegal, because there are named arguments before positional ones: -.. include:: include/snippets/functions/signature_plain_python_call_illegal.py +.. include:: include/snippets/functions/signature_plain_python_call_illegal.illpy :code: python Finally, python knows the concept of ``*args`` and ``**kwargs``, which allow one to define a function that accepts a variable number of positional and keyword arguments (also known as a _variadic_ function). @@ -190,7 +190,7 @@ This particularly useful when exposing a process function in a wrapping workchai The user can now access the input description directly through the spec of the work chain, without having to go to the process function itself. For example, in an interactive shell: -.. include:: include/snippets/functions/parse_docstring_expose_ipython.py +.. include:: include/snippets/functions/parse_docstring_expose_ipython.illpy :code: ipython Return values diff --git a/docs/source/topics/processes/include/snippets/functions/parse_docstring_expose_ipython.py b/docs/source/topics/processes/include/snippets/functions/parse_docstring_expose_ipython.illpy similarity index 100% rename from docs/source/topics/processes/include/snippets/functions/parse_docstring_expose_ipython.py rename to docs/source/topics/processes/include/snippets/functions/parse_docstring_expose_ipython.illpy diff --git a/docs/source/topics/processes/include/snippets/functions/signature_plain_python_call_illegal.py b/docs/source/topics/processes/include/snippets/functions/signature_plain_python_call_illegal.illpy similarity index 100% rename from docs/source/topics/processes/include/snippets/functions/signature_plain_python_call_illegal.py rename to docs/source/topics/processes/include/snippets/functions/signature_plain_python_call_illegal.illpy diff --git a/src/aiida/cmdline/commands/cmd_profile.py b/src/aiida/cmdline/commands/cmd_profile.py index 3dd21b56bf..367f234fd5 100644 --- a/src/aiida/cmdline/commands/cmd_profile.py +++ b/src/aiida/cmdline/commands/cmd_profile.py @@ -10,6 +10,8 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import click from aiida.cmdline.commands.cmd_verdi import verdi @@ -18,7 +20,10 @@ from aiida.cmdline.params.options.commands import setup from aiida.cmdline.utils import defaults, echo from aiida.common import exceptions -from aiida.manage.configuration import Profile, create_profile, get_config +from aiida.manage.configuration.config import get_config + +if TYPE_CHECKING: + from aiida.manage.configuration import Profile @verdi.group('profile') @@ -56,6 +61,7 @@ def command_create_profile( """ from aiida.brokers.rabbitmq.defaults import detect_rabbitmq_config from aiida.common import docs + from aiida.manage.configuration import create_profile from aiida.plugins.entry_point import get_entry_point_from_class if not storage_cls.read_only and email is None: diff --git a/src/aiida/manage/__init__.py b/src/aiida/manage/__init__.py index e5ff00cbdb..6d23786696 100644 --- a/src/aiida/manage/__init__.py +++ b/src/aiida/manage/__init__.py @@ -17,11 +17,6 @@ .. note:: Modules in this sub package may require the database environment to be loaded """ - -# AUTO-GENERATED - -# fmt: off - from .caching import * from .configuration import * from .external import * @@ -47,4 +42,3 @@ 'upgrade_config', ) -# fmt: on diff --git a/src/aiida/manage/configuration/__init__.py b/src/aiida/manage/configuration/__init__.py index 0470089546..3d5d42013a 100644 --- a/src/aiida/manage/configuration/__init__.py +++ b/src/aiida/manage/configuration/__init__.py @@ -11,8 +11,7 @@ from __future__ import annotations -# AUTO-GENERATED -# fmt: off +from .config import * from .migrations import * from .options import * from .profile import * @@ -31,294 +30,13 @@ 'get_option_names', 'parse_option', 'upgrade_config', -) - -# fmt: on - -# END AUTO-GENERATED - - -__all__ += ( + 'create_profile', + 'get_profile', + 'load_profile', + 'profile_context', 'get_config', 'get_config_option', 'get_config_path', - 'get_profile', - 'load_profile', 'reset_config', 'CONFIG', ) - -import os -import warnings -from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, Optional - -from aiida.common.warnings import AiidaDeprecationWarning - -if TYPE_CHECKING: - from aiida.orm import User - - from .config import Config - -# global variables for aiida -CONFIG: Optional['Config'] = None - - -def get_config_path(): - """Returns path to .aiida configuration directory.""" - from .settings import AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME - - return os.path.join(AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME) - - -def load_config(create=False) -> 'Config': - """Instantiate Config object representing an AiiDA configuration file. - - Warning: Contrary to :func:`~aiida.manage.configuration.get_config`, this function is uncached and will always - create a new Config object. You may want to call :func:`~aiida.manage.configuration.get_config` instead. - - :param create: if True, will create the configuration file if it does not already exist - :type create: bool - - :return: the config - :rtype: :class:`~aiida.manage.configuration.config.Config` - :raises aiida.common.MissingConfigurationError: if the configuration file could not be found and create=False - """ - from aiida.common import exceptions - - from .config import Config - - filepath = get_config_path() - - if not os.path.isfile(filepath) and not create: - raise exceptions.MissingConfigurationError(f'configuration file {filepath} does not exist') - - try: - config = Config.from_file(filepath) - except ValueError as exc: - raise exceptions.ConfigurationError(f'configuration file {filepath} contains invalid JSON') from exc - - _merge_deprecated_cache_yaml(config, filepath) - - return config - - -def _merge_deprecated_cache_yaml(config, filepath): - """Merge the deprecated cache_config.yml into the config.""" - cache_path = os.path.join(os.path.dirname(filepath), 'cache_config.yml') - if not os.path.exists(cache_path): - return - - # Imports are here to avoid them when not needed - import shutil - - import yaml - - from aiida.common import timezone - - cache_path_backup = None - # Keep generating a new backup filename based on the current time until it does not exist - while not cache_path_backup or os.path.isfile(cache_path_backup): - cache_path_backup = f"{cache_path}.{timezone.now().strftime('%Y%m%d-%H%M%S.%f')}" - - warnings.warn( - 'cache_config.yml use is deprecated and support will be removed in `v3.0`. Merging into config.json and ' - f'moving to: {cache_path_backup}', - AiidaDeprecationWarning, - stacklevel=2, - ) - - with open(cache_path, 'r', encoding='utf8') as handle: - cache_config = yaml.safe_load(handle) - for profile_name, data in cache_config.items(): - if profile_name not in config.profile_names: - warnings.warn(f"Profile '{profile_name}' from cache_config.yml not in config.json, skipping", UserWarning) - continue - for key, option_name in [ - ('default', 'caching.default_enabled'), - ('enabled', 'caching.enabled_for'), - ('disabled', 'caching.disabled_for'), - ]: - if key in data: - value = data[key] - # in case of empty key - value = [] if value is None and key != 'default' else value - config.set_option(option_name, value, scope=profile_name) - config.store() - shutil.move(cache_path, cache_path_backup) - - -def load_profile(profile: Optional[str] = None, allow_switch=False) -> 'Profile': - """Load a global profile, unloading any previously loaded profile. - - .. note:: if a profile is already loaded and no explicit profile is specified, nothing will be done - - :param profile: the name of the profile to load, by default will use the one marked as default in the config - :param allow_switch: if True, will allow switching to a different profile when storage is already loaded - - :return: the loaded `Profile` instance - :raises `aiida.common.exceptions.InvalidOperation`: - if another profile has already been loaded and allow_switch is False - """ - from aiida.manage import get_manager - - return get_manager().load_profile(profile, allow_switch) - - -def get_profile() -> Optional['Profile']: - """Return the currently loaded profile. - - :return: the globally loaded `Profile` instance or `None` - """ - from aiida.manage import get_manager - - return get_manager().get_profile() - - -@contextmanager -def profile_context(profile: 'Profile' | str | None = None, allow_switch=False) -> 'Profile': - """Return a context manager for temporarily loading a profile, and unloading on exit. - - :param profile: the name of the profile to load, by default will use the one marked as default in the config - :param allow_switch: if True, will allow switching to a different profile - - :return: a context manager for temporarily loading a profile - """ - from aiida.manage import get_manager - - manager = get_manager() - current_profile = manager.get_profile() - yield manager.load_profile(profile, allow_switch) - if current_profile is None: - manager.unload_profile() - else: - manager.load_profile(current_profile, allow_switch=True) - - -def create_default_user( - profile: Profile, - email: str, - first_name: Optional[str] = None, - last_name: Optional[str] = None, - institution: Optional[str] = None, -) -> User: - """Create a default user for the given profile. - - If the profile's storage is read only, a random existing user will be queried and set as default. Otherwise a new - user is created with the provided details and set as user. - - :param profile: The profile to create the user in. - :param email: Email for the default user. - :param first_name: First name for the default user. - :param last_name: Last name for the default user. - :param institution: Institution for the default user. - :returns: The user that was set as the default user. - """ - from aiida.manage import get_manager - from aiida.orm import User - - with profile_context(profile, allow_switch=True): - manager = get_manager() - storage = manager.get_profile_storage() - - if storage.read_only: - # Check if the storage contains any users, and just set a random one as default user. - user = User.collection.query().first(flat=True) - else: - # Otherwise create a user and store it - user = User(email=email, first_name=first_name, last_name=last_name, institution=institution).store() - - # The user can be ``None`` if the storage is read-only and doesn't contain any users. This shouldn't happen in - # real situations, but this safe guard is added to be safe. - if user: - manager.set_default_user_email(profile, user.email) - - return user - - -def create_profile( - config: 'Config', - *, - storage_backend: str, - storage_config: dict[str, Any], - broker_backend: str | None = None, - broker_config: dict[str, Any] | None = None, - name: str, - email: str, - first_name: Optional[str] = None, - last_name: Optional[str] = None, - institution: Optional[str] = None, - is_test_profile: bool = False, -) -> Profile: - """Create a new profile, initialise its storage and create a default user. - - :param config: The config instance. - :param name: Name of the profile. - :param email: Email for the default user. - :param first_name: First name for the default user. - :param last_name: Last name for the default user. - :param institution: Institution for the default user. - :param create_user: If `True`, creates a user that is set as the default user. - :param storage_backend: The entry point to the :class:`aiida.orm.implementation.storage_backend.StorageBackend` - implementation to use for the storage. - :param storage_config: The configuration necessary to initialise and connect to the storage backend. - :param broker_backend: The entry point to the :class:`aiida.brokers.Broker` implementation to use for the broker. - :param broker_config: The configuration necessary to initialise and connect to the broker. - """ - profile: Profile = config.create_profile( - name=name, - storage_backend=storage_backend, - storage_config=storage_config, - broker_backend=broker_backend, - broker_config=broker_config, - is_test_profile=is_test_profile, - ) - - create_default_user(profile, email, first_name, last_name, institution) - - return profile - - -def reset_config(): - """Reset the globally loaded config. - - .. warning:: This is experimental functionality and should for now be used only internally. If the reset is unclean - weird unknown side-effects may occur that end up corrupting or destroying data. - """ - global CONFIG # noqa: PLW0603 - CONFIG = None - - -def get_config(create=False): - """Return the current configuration. - - If the configuration has not been loaded yet - * the configuration is loaded using ``load_config`` - * the global `CONFIG` variable is set - * the configuration object is returned - - Note: This function will except if no configuration file can be found. Only call this function, if you need - information from the configuration file. - - :param create: if True, will create the configuration file if it does not already exist - :type create: bool - - :return: the config - :rtype: :class:`~aiida.manage.configuration.config.Config` - :raises aiida.common.ConfigurationError: if the configuration file could not be found, read or deserialized - """ - global CONFIG # noqa: PLW0603 - - if not CONFIG: - CONFIG = load_config(create=create) - - if CONFIG.get_option('warnings.showdeprecations'): - # If the user does not want to get AiiDA deprecation warnings, we disable them - this can be achieved with:: - # verdi config warnings.showdeprecations False - # Note that the AiidaDeprecationWarning does NOT inherit from DeprecationWarning - warnings.simplefilter('default', AiidaDeprecationWarning) - # This should default to 'once', i.e. once per different message - else: - warnings.simplefilter('ignore', AiidaDeprecationWarning) - - return CONFIG diff --git a/src/aiida/manage/configuration/config.py b/src/aiida/manage/configuration/config.py index d5033a6b56..426a5e9a15 100644 --- a/src/aiida/manage/configuration/config.py +++ b/src/aiida/manage/configuration/config.py @@ -20,187 +20,31 @@ import io import json import os -import uuid -from typing import Any, Dict, List, Optional, Tuple - -from pydantic import ( - BaseModel, - ConfigDict, - Field, - ValidationError, - field_serializer, - field_validator, -) +import warnings +from typing import Any, Dict, Optional + +from pydantic import ValidationError from aiida.common.exceptions import ConfigurationError, EntryPointError, StorageMigrationError -from aiida.common.log import AIIDA_LOGGER, LogLevels +from aiida.common.log import AIIDA_LOGGER +from aiida.common.warnings import AiidaDeprecationWarning from .options import Option, get_option, get_option_names, parse_option from .profile import Profile +from .schema import ConfigSchema + +__all__ = ( + 'get_config', + 'get_config_option', + 'get_config_path', + 'reset_config', + 'CONFIG', +) LOGGER = AIIDA_LOGGER.getChild('manage.configuration.config') - -class ConfigVersionSchema(BaseModel, defer_build=True): - """Schema for the version configuration of an AiiDA instance.""" - - CURRENT: int - OLDEST_COMPATIBLE: int - - -class ProfileOptionsSchema(BaseModel, defer_build=True): - """Schema for the options of an AiiDA profile.""" - - model_config = ConfigDict(use_enum_values=True) - - runner__poll__interval: int = Field(60, description='Polling interval in seconds to be used by process runners.') - daemon__default_workers: int = Field( - 1, description='Default number of workers to be launched by `verdi daemon start`.' - ) - daemon__timeout: int = Field( - 2, - description='Used to set default timeout in the `DaemonClient` for calls to the daemon.', - ) - daemon__worker_process_slots: int = Field( - 200, description='Maximum number of concurrent process tasks that each daemon worker can handle.' - ) - daemon__recursion_limit: int = Field(3000, description='Maximum recursion depth for the daemon workers.') - db__batch_size: int = Field( - 100000, - description='Batch size for bulk CREATE operations in the database. Avoids hitting MaxAllocSize of PostgreSQL ' - '(1GB) when creating large numbers of database records in one go.', - ) - verdi__shell__auto_import: str = Field( - ':', - description='Additional modules/functions/classes to be automatically loaded in `verdi shell`, split by `:`.', - ) - logging__aiida_loglevel: LogLevels = Field( - 'REPORT', description='Minimum level to log to daemon log and the `DbLog` table for the `aiida` logger.' - ) - logging__verdi_loglevel: LogLevels = Field( - 'REPORT', description='Minimum level to log to console when running a `verdi` command.' - ) - logging__disk_objectstore_loglevel: LogLevels = Field( - 'INFO', description='Minimum level to log to daemon log and the `DbLog` table for `disk_objectstore` logger.' - ) - logging__db_loglevel: LogLevels = Field('REPORT', description='Minimum level to log to the DbLog table.') - logging__plumpy_loglevel: LogLevels = Field( - 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `plumpy` logger.' - ) - logging__kiwipy_loglevel: LogLevels = Field( - 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `kiwipy` logger' - ) - logging__paramiko_loglevel: LogLevels = Field( - 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `paramiko` logger' - ) - logging__alembic_loglevel: LogLevels = Field( - 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `alembic` logger' - ) - logging__sqlalchemy_loglevel: LogLevels = Field( - 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `sqlalchemy` logger' - ) - logging__circus_loglevel: LogLevels = Field( - 'INFO', description='Minimum level to log to daemon log and the `DbLog` table for the `circus` logger' - ) - logging__aiopika_loglevel: LogLevels = Field( - 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `aiopika` logger' - ) - warnings__showdeprecations: bool = Field(True, description='Whether to print AiiDA deprecation warnings.') - warnings__rabbitmq_version: bool = Field( - True, description='Whether to print a warning when an incompatible version of RabbitMQ is configured.' - ) - transport__task_retry_initial_interval: int = Field( - 20, description='Initial time interval for the exponential backoff mechanism.' - ) - transport__task_maximum_attempts: int = Field( - 5, description='Maximum number of transport task attempts before a Process is Paused.' - ) - rmq__task_timeout: int = Field(10, description='Timeout in seconds for communications with RabbitMQ.') - storage__sandbox: Optional[str] = Field( - None, description='Absolute path to the directory to store sandbox folders.' - ) - caching__default_enabled: bool = Field(False, description='Enable calculation caching by default.') - caching__enabled_for: List[str] = Field([], description='Calculation entry points to enable caching on.') - caching__disabled_for: List[str] = Field([], description='Calculation entry points to disable caching on.') - - @field_validator('caching__enabled_for', 'caching__disabled_for') - @classmethod - def validate_caching_identifier_pattern(cls, value: List[str]) -> List[str]: - """Validate the caching identifier patterns.""" - from aiida.manage.caching import _validate_identifier_pattern - - for identifier in value: - _validate_identifier_pattern(identifier=identifier, strict=True) - return value - - -class GlobalOptionsSchema(ProfileOptionsSchema, defer_build=True): - """Schema for the global options of an AiiDA instance.""" - - autofill__user__email: Optional[str] = Field( - None, description='Default user email to use when creating new profiles.' - ) - autofill__user__first_name: Optional[str] = Field( - None, description='Default user first name to use when creating new profiles.' - ) - autofill__user__last_name: Optional[str] = Field( - None, description='Default user last name to use when creating new profiles.' - ) - autofill__user__institution: Optional[str] = Field( - None, description='Default user institution to use when creating new profiles.' - ) - rest_api__profile_switching: bool = Field( - False, description='Toggle whether the profile can be specified in requests submitted to the REST API.' - ) - warnings__development_version: bool = Field( - True, - description='Whether to print a warning when a profile is loaded while a development version is installed.', - ) - - -class ProfileStorageConfig(BaseModel, defer_build=True): - """Schema for the storage backend configuration of an AiiDA profile.""" - - backend: str - config: Dict[str, Any] - - -class ProcessControlConfig(BaseModel, defer_build=True): - """Schema for the process control configuration of an AiiDA profile.""" - - broker_protocol: str = Field('amqp', description='Protocol for connecting to the message broker.') - broker_username: str = Field('guest', description='Username for message broker authentication.') - broker_password: str = Field('guest', description='Password for message broker.') - broker_host: str = Field('127.0.0.1', description='Hostname of the message broker.') - broker_port: int = Field(5432, description='Port of the message broker.') - broker_virtual_host: str = Field('', description='Virtual host to use for the message broker.') - broker_parameters: dict[str, Any] = Field( - default_factory=dict, description='Arguments to be encoded as query parameters.' - ) - - -class ProfileSchema(BaseModel, defer_build=True): - """Schema for the configuration of an AiiDA profile.""" - - uuid: str = Field(description='A UUID that uniquely identifies the profile.', default_factory=uuid.uuid4) - storage: ProfileStorageConfig - process_control: ProcessControlConfig - default_user_email: Optional[str] = None - test_profile: bool = False - options: Optional[ProfileOptionsSchema] = None - - @field_serializer('uuid') - def serialize_dt(self, value: uuid.UUID, _info): - return str(value) - - -class ConfigSchema(BaseModel, defer_build=True): - """Schema for the configuration of an AiiDA instance.""" - - CONFIG_VERSION: Optional[ConfigVersionSchema] = None - profiles: Optional[dict[str, ProfileSchema]] = None - options: Optional[GlobalOptionsSchema] = None - default_profile: Optional[str] = None +# global variables for aiida +CONFIG: Optional['Config'] = None class Config: @@ -694,7 +538,7 @@ def get_option(self, option_name, scope=None, default=True): return value - def get_options(self, scope: Optional[str] = None) -> Dict[str, Tuple[Option, str, Any]]: + def get_options(self, scope: Optional[str] = None) -> Dict[str, tuple[Option, str, Any]]: """Return a dictionary of all option values and their source ('profile', 'global', or 'default'). :param scope: the profile name or globally if not specified @@ -782,6 +626,135 @@ def _atomic_write(self, filepath=None): os.rename(handle.name, self.filepath) +def get_config_path(): + """Returns path to .aiida configuration directory.""" + from .settings import AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME + + return os.path.join(AIIDA_CONFIG_FOLDER, DEFAULT_CONFIG_FILE_NAME) + + +def load_config(create=False) -> 'Config': + """Instantiate Config object representing an AiiDA configuration file. + + Warning: Contrary to :func:`~aiida.manage.configuration.get_config`, this function is uncached and will always + create a new Config object. You may want to call :func:`~aiida.manage.configuration.get_config` instead. + + :param create: if True, will create the configuration file if it does not already exist + :type create: bool + + :return: the config + :rtype: :class:`~aiida.manage.configuration.config.Config` + :raises aiida.common.MissingConfigurationError: if the configuration file could not be found and create=False + """ + from aiida.common import exceptions + + from .config import Config + + filepath = get_config_path() + + if not os.path.isfile(filepath) and not create: + raise exceptions.MissingConfigurationError(f'configuration file {filepath} does not exist') + + try: + config = Config.from_file(filepath) + except ValueError as exc: + raise exceptions.ConfigurationError(f'configuration file {filepath} contains invalid JSON') from exc + + _merge_deprecated_cache_yaml(config, filepath) + + return config + + +def _merge_deprecated_cache_yaml(config, filepath): + """Merge the deprecated cache_config.yml into the config.""" + cache_path = os.path.join(os.path.dirname(filepath), 'cache_config.yml') + if not os.path.exists(cache_path): + return + + # Imports are here to avoid them when not needed + import shutil + + import yaml + + from aiida.common import timezone + + cache_path_backup = None + # Keep generating a new backup filename based on the current time until it does not exist + while not cache_path_backup or os.path.isfile(cache_path_backup): + cache_path_backup = f"{cache_path}.{timezone.now().strftime('%Y%m%d-%H%M%S.%f')}" + + warnings.warn( + 'cache_config.yml use is deprecated and support will be removed in `v3.0`. Merging into config.json and ' + f'moving to: {cache_path_backup}', + AiidaDeprecationWarning, + stacklevel=2, + ) + + with open(cache_path, 'r', encoding='utf8') as handle: + cache_config = yaml.safe_load(handle) + for profile_name, data in cache_config.items(): + if profile_name not in config.profile_names: + warnings.warn(f"Profile '{profile_name}' from cache_config.yml not in config.json, skipping", UserWarning) + continue + for key, option_name in [ + ('default', 'caching.default_enabled'), + ('enabled', 'caching.enabled_for'), + ('disabled', 'caching.disabled_for'), + ]: + if key in data: + value = data[key] + # in case of empty key + value = [] if value is None and key != 'default' else value + config.set_option(option_name, value, scope=profile_name) + config.store() + shutil.move(cache_path, cache_path_backup) + + +def reset_config(): + """Reset the globally loaded config. + + .. warning:: This is experimental functionality and should for now be used only internally. If the reset is unclean + weird unknown side-effects may occur that end up corrupting or destroying data. + """ + global CONFIG # noqa: PLW0603 + CONFIG = None + + +def get_config(create=False): + """Return the current configuration. + + If the configuration has not been loaded yet + * the configuration is loaded using ``load_config`` + * the global `CONFIG` variable is set + * the configuration object is returned + + Note: This function will except if no configuration file can be found. Only call this function, if you need + information from the configuration file. + + :param create: if True, will create the configuration file if it does not already exist + :type create: bool + + :return: the config + :rtype: :class:`~aiida.manage.configuration.config.Config` + :raises aiida.common.ConfigurationError: if the configuration file could not be found, read or deserialized + """ + global CONFIG # noqa: PLW0603 + + if not CONFIG: + CONFIG = load_config(create=create) + + if CONFIG.get_option('warnings.showdeprecations'): + # If the user does not want to get AiiDA deprecation warnings, we disable them - this can be achieved with:: + # verdi config warnings.showdeprecations False + # Note that the AiidaDeprecationWarning does NOT inherit from DeprecationWarning + warnings.simplefilter('default', AiidaDeprecationWarning) + # This should default to 'once', i.e. once per different message + else: + warnings.simplefilter('ignore', AiidaDeprecationWarning) + + return CONFIG + + def get_config_option(option_name: str) -> Any: """Return the value of a configuration option. diff --git a/src/aiida/manage/configuration/migrations/migrations.py b/src/aiida/manage/configuration/migrations/migrations.py index c432c6fe81..1c483a5962 100644 --- a/src/aiida/manage/configuration/migrations/migrations.py +++ b/src/aiida/manage/configuration/migrations/migrations.py @@ -113,7 +113,7 @@ class SimplifyDefaultProfiles(SingleMigration): up_compatible = 3 def upgrade(self, config: ConfigType) -> None: - from aiida.manage.configuration import get_profile + from aiida.manage.configuration.profile import get_profile global_profile = get_profile() default_profiles = config.pop('default_profiles', None) diff --git a/src/aiida/manage/configuration/options.py b/src/aiida/manage/configuration/options.py index 110df675f6..8cd8989c55 100644 --- a/src/aiida/manage/configuration/options.py +++ b/src/aiida/manage/configuration/options.py @@ -8,7 +8,7 @@ ########################################################################### """Definition of known configuration options and methods to parse and get option values.""" -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List from aiida.common.exceptions import ConfigurationError @@ -48,7 +48,7 @@ def description(self) -> str: @property def global_only(self) -> bool: - from .config import ProfileOptionsSchema + from .schema import ProfileOptionsSchema return self._name.replace('.', '__') not in ProfileOptionsSchema.model_fields @@ -61,7 +61,7 @@ def validate(self, value: Any) -> Any: """ from pydantic import ValidationError - from .config import GlobalOptionsSchema + from .schema import GlobalOptionsSchema attribute = self.name.replace('.', '__') @@ -87,14 +87,14 @@ def validate(self, value: Any) -> Any: def get_option_names() -> List[str]: """Return a list of available option names.""" - from .config import GlobalOptionsSchema + from .schema import GlobalOptionsSchema return [key.replace('__', '.') for key in GlobalOptionsSchema.model_fields] def get_option(name: str) -> Option: """Return option.""" - from .config import GlobalOptionsSchema + from .schema import GlobalOptionsSchema options = GlobalOptionsSchema.model_fields option_name = name.replace('.', '__') @@ -103,7 +103,7 @@ def get_option(name: str) -> Option: return Option(name, GlobalOptionsSchema.model_json_schema()['properties'][option_name], options[option_name]) -def parse_option(option_name: str, option_value: Any) -> Tuple[Option, Any]: +def parse_option(option_name: str, option_value: Any) -> tuple[Option, Any]: """Parse and validate a value for a configuration option. :param option_name: the name of the configuration option diff --git a/src/aiida/manage/configuration/profile.py b/src/aiida/manage/configuration/profile.py index acaca2e892..b208bf5a08 100644 --- a/src/aiida/manage/configuration/profile.py +++ b/src/aiida/manage/configuration/profile.py @@ -13,6 +13,7 @@ import collections import os import pathlib +from contextlib import contextmanager from copy import deepcopy from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Type @@ -21,9 +22,18 @@ from .options import parse_option if TYPE_CHECKING: + from aiida.orm import User from aiida.orm.implementation import StorageBackend -__all__ = ('Profile',) + from .config import Config + +__all__ = ( + 'Profile', + 'create_profile', + 'get_profile', + 'load_profile', + 'profile_context', +) class Profile: @@ -254,3 +264,134 @@ def filepaths(self): 'pid': str(DAEMON_DIR / f'aiida-{self.name}.pid'), }, } + + +def create_default_user( + profile: Profile, + email: str, + first_name: Optional[str] = None, + last_name: Optional[str] = None, + institution: Optional[str] = None, +) -> User: + """Create a default user for the given profile. + + If the profile's storage is read only, a random existing user will be queried and set as default. Otherwise a new + user is created with the provided details and set as user. + + :param profile: The profile to create the user in. + :param email: Email for the default user. + :param first_name: First name for the default user. + :param last_name: Last name for the default user. + :param institution: Institution for the default user. + :returns: The user that was set as the default user. + """ + from aiida.manage import get_manager + from aiida.orm import User + + with profile_context(profile, allow_switch=True): + manager = get_manager() + storage = manager.get_profile_storage() + + if storage.read_only: + # Check if the storage contains any users, and just set a random one as default user. + user = User.collection.query().first(flat=True) + else: + # Otherwise create a user and store it + user = User(email=email, first_name=first_name, last_name=last_name, institution=institution).store() + + # The user can be ``None`` if the storage is read-only and doesn't contain any users. This shouldn't happen in + # real situations, but this safe guard is added to be safe. + if user: + manager.set_default_user_email(profile, user.email) + + return user + + +def create_profile( + config: 'Config', + *, + storage_backend: str, + storage_config: dict[str, Any], + broker_backend: str | None = None, + broker_config: dict[str, Any] | None = None, + name: str, + email: str, + first_name: Optional[str] = None, + last_name: Optional[str] = None, + institution: Optional[str] = None, + is_test_profile: bool = False, +) -> Profile: + """Create a new profile, initialise its storage and create a default user. + + :param config: The config instance. + :param name: Name of the profile. + :param email: Email for the default user. + :param first_name: First name for the default user. + :param last_name: Last name for the default user. + :param institution: Institution for the default user. + :param create_user: If `True`, creates a user that is set as the default user. + :param storage_backend: The entry point to the :class:`aiida.orm.implementation.storage_backend.StorageBackend` + implementation to use for the storage. + :param storage_config: The configuration necessary to initialise and connect to the storage backend. + :param broker_backend: The entry point to the :class:`aiida.brokers.Broker` implementation to use for the broker. + :param broker_config: The configuration necessary to initialise and connect to the broker. + """ + profile: Profile = config.create_profile( + name=name, + storage_backend=storage_backend, + storage_config=storage_config, + broker_backend=broker_backend, + broker_config=broker_config, + is_test_profile=is_test_profile, + ) + + create_default_user(profile, email, first_name, last_name, institution) + + return profile + + +def load_profile(profile: Optional[str] = None, allow_switch=False) -> 'Profile': + """Load a global profile, unloading any previously loaded profile. + + .. note:: if a profile is already loaded and no explicit profile is specified, nothing will be done + + :param profile: the name of the profile to load, by default will use the one marked as default in the config + :param allow_switch: if True, will allow switching to a different profile when storage is already loaded + + :return: the loaded `Profile` instance + :raises `aiida.common.exceptions.InvalidOperation`: + if another profile has already been loaded and allow_switch is False + """ + from aiida.manage import get_manager + + return get_manager().load_profile(profile, allow_switch) + + +def get_profile() -> Optional['Profile']: + """Return the currently loaded profile. + + :return: the globally loaded `Profile` instance or `None` + """ + from aiida.manage import get_manager + + return get_manager().get_profile() + + +@contextmanager +def profile_context(profile: 'Profile' | str | None = None, allow_switch=False) -> 'Profile': + """Return a context manager for temporarily loading a profile, and unloading on exit. + + :param profile: the name of the profile to load, by default will use the one marked as default in the config + :param allow_switch: if True, will allow switching to a different profile + + :return: a context manager for temporarily loading a profile + """ + from aiida.manage import get_manager + + manager = get_manager() + current_profile = manager.get_profile() + yield manager.load_profile(profile, allow_switch) + if current_profile is None: + manager.unload_profile() + else: + manager.load_profile(current_profile, allow_switch=True) diff --git a/src/aiida/manage/configuration/schema/__init__.py b/src/aiida/manage/configuration/schema/__init__.py index c56ff0a1f8..572266a264 100644 --- a/src/aiida/manage/configuration/schema/__init__.py +++ b/src/aiida/manage/configuration/schema/__init__.py @@ -6,3 +6,176 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator + +from aiida.common.log import LogLevels + +if TYPE_CHECKING: + from uuid import UUID + + +class ConfigVersionSchema(BaseModel, defer_build=True): + """Schema for the version configuration of an AiiDA instance.""" + + CURRENT: int + OLDEST_COMPATIBLE: int + + +class ProfileOptionsSchema(BaseModel, defer_build=True): + """Schema for the options of an AiiDA profile.""" + + model_config = ConfigDict(use_enum_values=True) + + runner__poll__interval: int = Field(60, description='Polling interval in seconds to be used by process runners.') + daemon__default_workers: int = Field( + 1, description='Default number of workers to be launched by `verdi daemon start`.' + ) + daemon__timeout: int = Field( + 2, + description='Used to set default timeout in the `DaemonClient` for calls to the daemon.', + ) + daemon__worker_process_slots: int = Field( + 200, description='Maximum number of concurrent process tasks that each daemon worker can handle.' + ) + daemon__recursion_limit: int = Field(3000, description='Maximum recursion depth for the daemon workers.') + db__batch_size: int = Field( + 100000, + description='Batch size for bulk CREATE operations in the database. Avoids hitting MaxAllocSize of PostgreSQL ' + '(1GB) when creating large numbers of database records in one go.', + ) + verdi__shell__auto_import: str = Field( + ':', + description='Additional modules/functions/classes to be automatically loaded in `verdi shell`, split by `:`.', + ) + logging__aiida_loglevel: LogLevels = Field( + 'REPORT', description='Minimum level to log to daemon log and the `DbLog` table for the `aiida` logger.' + ) + logging__verdi_loglevel: LogLevels = Field( + 'REPORT', description='Minimum level to log to console when running a `verdi` command.' + ) + logging__disk_objectstore_loglevel: LogLevels = Field( + 'INFO', description='Minimum level to log to daemon log and the `DbLog` table for `disk_objectstore` logger.' + ) + logging__db_loglevel: LogLevels = Field('REPORT', description='Minimum level to log to the DbLog table.') + logging__plumpy_loglevel: LogLevels = Field( + 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `plumpy` logger.' + ) + logging__kiwipy_loglevel: LogLevels = Field( + 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `kiwipy` logger' + ) + logging__paramiko_loglevel: LogLevels = Field( + 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `paramiko` logger' + ) + logging__alembic_loglevel: LogLevels = Field( + 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `alembic` logger' + ) + logging__sqlalchemy_loglevel: LogLevels = Field( + 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `sqlalchemy` logger' + ) + logging__circus_loglevel: LogLevels = Field( + 'INFO', description='Minimum level to log to daemon log and the `DbLog` table for the `circus` logger' + ) + logging__aiopika_loglevel: LogLevels = Field( + 'WARNING', description='Minimum level to log to daemon log and the `DbLog` table for the `aiopika` logger' + ) + warnings__showdeprecations: bool = Field(True, description='Whether to print AiiDA deprecation warnings.') + warnings__rabbitmq_version: bool = Field( + True, description='Whether to print a warning when an incompatible version of RabbitMQ is configured.' + ) + transport__task_retry_initial_interval: int = Field( + 20, description='Initial time interval for the exponential backoff mechanism.' + ) + transport__task_maximum_attempts: int = Field( + 5, description='Maximum number of transport task attempts before a Process is Paused.' + ) + rmq__task_timeout: int = Field(10, description='Timeout in seconds for communications with RabbitMQ.') + storage__sandbox: Optional[str] = Field( + None, description='Absolute path to the directory to store sandbox folders.' + ) + caching__default_enabled: bool = Field(False, description='Enable calculation caching by default.') + caching__enabled_for: list[str] = Field([], description='Calculation entry points to enable caching on.') + caching__disabled_for: list[str] = Field([], description='Calculation entry points to disable caching on.') + + @field_validator('caching__enabled_for', 'caching__disabled_for') + @classmethod + def validate_caching_identifier_pattern(cls, value: list[str]) -> list[str]: + """Validate the caching identifier patterns.""" + from aiida.manage.caching import _validate_identifier_pattern + + for identifier in value: + _validate_identifier_pattern(identifier=identifier, strict=True) + return value + + +class GlobalOptionsSchema(ProfileOptionsSchema, defer_build=True): + """Schema for the global options of an AiiDA instance.""" + + autofill__user__email: Optional[str] = Field( + None, description='Default user email to use when creating new profiles.' + ) + autofill__user__first_name: Optional[str] = Field( + None, description='Default user first name to use when creating new profiles.' + ) + autofill__user__last_name: Optional[str] = Field( + None, description='Default user last name to use when creating new profiles.' + ) + autofill__user__institution: Optional[str] = Field( + None, description='Default user institution to use when creating new profiles.' + ) + rest_api__profile_switching: bool = Field( + False, description='Toggle whether the profile can be specified in requests submitted to the REST API.' + ) + warnings__development_version: bool = Field( + True, + description='Whether to print a warning when a profile is loaded while a development version is installed.', + ) + + +class ProfileStorageConfig(BaseModel, defer_build=True): + """Schema for the storage backend configuration of an AiiDA profile.""" + + backend: str + config: dict[str, Any] + + +class ProcessControlConfig(BaseModel, defer_build=True): + """Schema for the process control configuration of an AiiDA profile.""" + + broker_protocol: str = Field('amqp', description='Protocol for connecting to the message broker.') + broker_username: str = Field('guest', description='Username for message broker authentication.') + broker_password: str = Field('guest', description='Password for message broker.') + broker_host: str = Field('127.0.0.1', description='Hostname of the message broker.') + broker_port: int = Field(5432, description='Port of the message broker.') + broker_virtual_host: str = Field('', description='Virtual host to use for the message broker.') + broker_parameters: dict[str, Any] = Field( + default_factory=dict, description='Arguments to be encoded as query parameters.' + ) + + +class ProfileSchema(BaseModel, defer_build=True): + """Schema for the configuration of an AiiDA profile.""" + + uuid: str = Field(description='A UUID that uniquely identifies the profile.', default_factory=uuid4) + storage: ProfileStorageConfig + process_control: ProcessControlConfig + default_user_email: Optional[str] = None + test_profile: bool = False + options: Optional[ProfileOptionsSchema] = None + + @field_serializer('uuid') + def serialize_dt(self, value: UUID, _info): + return str(value) + + +class ConfigSchema(BaseModel, defer_build=True): + """Schema for the configuration of an AiiDA instance.""" + + CONFIG_VERSION: Optional[ConfigVersionSchema] = None + profiles: Optional[dict[str, ProfileSchema]] = None + options: Optional[GlobalOptionsSchema] = None + default_profile: Optional[str] = None diff --git a/src/aiida/tools/pytest_fixtures/configuration.py b/src/aiida/tools/pytest_fixtures/configuration.py index ca38db3fb0..cf0dee5f17 100644 --- a/src/aiida/tools/pytest_fixtures/configuration.py +++ b/src/aiida/tools/pytest_fixtures/configuration.py @@ -111,7 +111,7 @@ def factory( name: str | None = None, email: str = 'test@localhost', ): - from aiida.manage.configuration import create_profile, profile_context + from aiida.manage.configuration.profile import create_profile, profile_context from aiida.manage.manager import get_manager manager = get_manager() diff --git a/tests/manage/configuration/test_options.py b/tests/manage/configuration/test_options.py index 7a76ceeec5..4793721cbc 100644 --- a/tests/manage/configuration/test_options.py +++ b/tests/manage/configuration/test_options.py @@ -12,8 +12,8 @@ from aiida import get_profile from aiida.common.exceptions import ConfigurationError from aiida.manage.configuration import get_config, get_config_option -from aiida.manage.configuration.config import GlobalOptionsSchema from aiida.manage.configuration.options import Option, get_option, get_option_names, parse_option +from aiida.manage.configuration.schema import GlobalOptionsSchema @pytest.mark.usefixtures('config_with_profile')