Feature/install from folder

README.md

@@ -154,18 +154,16 @@ aiida-pseudo install family <ARCHIVE> <LABEL>
 ```
 where `<ARCHIVE>` should be replaced with the pseudopotential archive and `<LABEL>` with the label to give to the family.
 The command will attempt to automatically detect the compression format of the archive.
-If this fails, you can specify the format manually with the `--archive-format/-F` option, for example, for a `.tar.gz` archive:
+If this fails, you can specify the format manually with the `--archive-format/-f` option, for example, for a `.tar.gz` archive:
 ```
-aiida-pseudo install family <ARCHIVE> <LABEL> -F gztar
+aiida-pseudo install family <ARCHIVE> <LABEL> -f gztar
 ```
 By default, the command will create a family of the base pseudopotential family type.
-If you want to create a more specific family, for example an `SsspFamily`, or a `PseudoDojoFamily`, you can provide the corresponding entry point to the `--family-type/-T` option:
+If you want to create a more specific family, for example an `CutoffsPseudoPotentialFamily`, you can provide the corresponding entry point to the `--family-type/-F` option:
 ```
-aiida-pseudo install family <ARCHIVE> <LABEL> -T pseudo.family.sssp
+aiida-pseudo install family <ARCHIVE> <LABEL> -F pseudo.family.cutoffs
 ```
-or
-```
-aiida-pseudo install family <ARCHIVE> <LABEL> -T pseudo.family.pseudo_dojo
+Note that the `pseudo.family.sssp` and `pseudo.family.pseudo_dojo` family types are blacklisted since they have their own dedicated install commands in `aiida-pseudo install sssp` and `aiida-pseudo install pseudo-dojo`, respectively.
 ```
 The available pseudopotential family classes can be listed with the command:
 ```

aiida_pseudo/cli/install.py

@@ -1,19 +1,18 @@
 # -*- coding: utf-8 -*-
 """Command to install a pseudo potential family."""
 import json
-import os
 import pathlib
 import shutil
 import tempfile
 
 import click
+import requests
 
 from aiida.cmdline.utils import decorators, echo
 from aiida.cmdline.params import options as options_core
-from aiida.cmdline.params import types
 
 from aiida_pseudo.groups.family import PseudoDojoConfiguration, SsspConfiguration
-from .params import options
+from .params import options, types
 from .root import cmd_root
 
 
@@ -23,46 +22,74 @@ def cmd_install():
 
 
 @cmd_install.command('family')
-@click.argument('archive', type=types.FileOrUrl(mode='rb'))
+@click.argument('archive', type=types.PathOrUrl(exists=True, file_okay=True))
 @click.argument('label', type=click.STRING)
 @options_core.DESCRIPTION(help='Description for the family.')
 @options.ARCHIVE_FORMAT()
-@options.FAMILY_TYPE()
+@options.FAMILY_TYPE(
+    type=types.PseudoPotentialFamilyTypeParam(exclude=('pseudo.family.sssp', 'pseudo.family.pseudo_dojo'))
+)
+@options.PSEUDO_TYPE()
 @options.TRACEBACK()
 @decorators.with_dbenv()
-def cmd_install_family(archive, label, description, archive_format, family_type, traceback):  # pylint: disable=too-many-arguments
-    """Install a standard pseudo potential family from an ARCHIVE on the local file system or from a URL.
-
-    The command will attempt to infer the archive format from the filename extension of the ARCHIVE. If this fails, the
-    archive format can be specified explicitly with the archive format option, which will also display which formats
-    are supported.
-
-    By default, the command will create a base `PseudoPotentialFamily`, but the type can be changed with the family
-    type option. If the base type is used, the pseudo potential files in the archive *have* to have filenames that
-    strictly follow the format `ELEMENT.EXTENSION`, because otherwise the element cannot be determined automatically.
+def cmd_install_family(archive, label, description, archive_format, family_type, pseudo_type, traceback):  # pylint: disable=too-many-arguments
+    """Install a standard pseudopotential family from an ARCHIVE.
+
+    The ARCHIVE can be a (compressed) archive of a directory containing the pseudopotentials on the local file system or
+    provided by an HTTP URL. Alternatively, it can be a normal directory on the local file system. The (unarchived)
+    directory should only contain the pseudopotential files and they cannot be in any subdirectory. In addition,
+    depending on the chosen pseudopotential type (see the option `-P/--pseudo-type`) there can be additional
+    requirements on the pseudopotential file and filename format.
+
+    If the ARCHIVE corresponds to a (compressed) archive, the command will attempt to infer the archive format from the
+    filename extension of the ARCHIVE. If this fails, the archive format can be specified explicitly with the archive
+    format option `-f/--archive-format`, which will also display which formats are supported. These format suffixes
+    follow the naming of the `shutil.unpack_archive` standard library method.
+
+    Once the ARCHIVE is downloaded, uncompressed and unarchived into a directory on the local file system, the command
+    will create a `PseudoPotentialFamily` instance where the type of the pseudopotential data nodes that are stored
+    within it is set through the `-P/--pseudo-type` option. If the default, `pseudo` (which corresponds to the data
+    plugin `PseudoPotentialData`), is used, the pseudopotential files in the archive *have* to have filenames that
+    strictly follow the format `ELEMENT.EXTENSION`, or the creation of the family will fail. This is because for the
+    default pseudopotential type, the format of the file is unknown and the family requires the element to be known,
+    which in this case can then only be parsed from the filename.
+
+    The pseudopotential family type that is created can also be changed with the `-F/--family-type` option. Note,
+    however, that not all values are accepted. For example, the `pseudo.family.sssp` and `pseudo.family.pseudo_dojo` are
+    blacklisted since they have their own dedicated commands in `install sssp` and `install pseudo-dojo`, respectively.
     """
     from .utils import attempt, create_family_from_archive
 
-    # The `archive` is now either a `http.client.HTTPResponse` or a normal filelike object, so we get the original file
-    # name in a different way.
-    try:
-        suffix = os.path.basename(archive.url)
-    except AttributeError:
-        suffix = os.path.basename(archive.name)
-
-    # Write the content of the archive to a temporary file, because `create_family_from_archive` does currently not
-    # accept filelike objects because the underlying `shutil.unpack_archive` does not. Likewise, `unpack_archive` will
-    # attempt to deduce the archive format from the filename extension, so it is important we maintain the original
-    # filename. Of course if this fails, users can specify the archive format explicitly wiht the corresponding option.
-    with tempfile.NamedTemporaryFile(mode='w+b', suffix=suffix) as handle:
-        shutil.copyfileobj(archive, handle)
-        handle.flush()
-
+    if isinstance(archive, pathlib.Path) and archive.is_dir():
+        with attempt(f'creating a pseudopotential family from directory `{archive}`...', include_traceback=traceback):
+            family = family_type.create_from_folder(archive, label, pseudo_type=pseudo_type)
+    elif isinstance(archive, pathlib.Path) and archive.is_file():
         with attempt('unpacking archive and parsing pseudos... ', include_traceback=traceback):
-            family = create_family_from_archive(family_type, label, pathlib.Path(handle.name), fmt=archive_format)
+            family = create_family_from_archive(
+                family_type, label, archive, fmt=archive_format, pseudo_type=pseudo_type
+            )
+    else:
+        # At this point, we can assume that it is not a valid filepath on disk, but rather a URL and the ``archive``
+        # variable will contain the result objects from the ``requests`` library. The validation of the URL will already
+        # have been done by the ``PathOrUrl`` parameter type, so the URL is reachable. The content of the URL must be
+        # copied to a local temporary file because `create_family_from_archive` does currently not accept filelike
+        # objects, because in turn the underlying `shutil.unpack_archive` does not. In addition, `unpack_archive` will
+        # attempt to deduce the archive format from the filename extension, so it is important we maintain the original
+        # filename. Of course if this fails, users can specify the archive format explicitly with the corresponding
+        # option. We get the filename by converting the URL to a ``Path`` object and taking the filename, using that as
+        # a suffix for the temporary file that is generated on disk to copy the content to.
+        suffix = pathlib.Path(archive.url).name
+        with tempfile.NamedTemporaryFile(mode='w+b', suffix=suffix) as handle:
+            handle.write(archive.content)
+            handle.flush()
+
+            with attempt('unpacking archive and parsing pseudos... ', include_traceback=traceback):
+                family = create_family_from_archive(
+                    family_type, label, pathlib.Path(handle.name), fmt=archive_format, pseudo_type=pseudo_type
+                )
 
     family.description = description
-    echo.echo_success(f'installed `{label}` containing {family.count()} pseudo potentials')
+    echo.echo_success(f'installed `{label}` containing {family.count()} pseudopotentials')
 
 
 def download_sssp(
@@ -78,23 +105,21 @@ def download_sssp(
     :param filepath_metadata: absolute filepath to write the metadata file to.
     :param traceback: boolean, if true, print the traceback when an exception occurs.
     """
-    import requests
-
     from aiida_pseudo.groups.family import SsspFamily
     from .utils import attempt
 
     url_sssp_base = 'https://legacy-archive.materialscloud.org/file/2018.0001/v4/'
     url_archive = f"{url_sssp_base}/{SsspFamily.format_configuration_filename(configuration, 'tar.gz')}"
     url_metadata = f"{url_sssp_base}/{SsspFamily.format_configuration_filename(configuration, 'json')}"
 
-    with attempt('downloading selected pseudo potentials archive... ', include_traceback=traceback):
+    with attempt('downloading selected pseudopotentials archive... ', include_traceback=traceback):
         response = requests.get(url_archive)
         response.raise_for_status()
         with open(filepath_archive, 'wb') as handle:
             handle.write(response.content)
             handle.flush()
 
-    with attempt('downloading selected pseudo potentials metadata... ', include_traceback=traceback):
+    with attempt('downloading selected pseudopotentials metadata... ', include_traceback=traceback):
         response = requests.get(url_metadata)
         response.raise_for_status()
         with open(filepath_metadata, 'wb') as handle:
@@ -115,23 +140,21 @@ def download_pseudo_dojo(
     :param filepath_metadata: absolute filepath to write the metadata archive to.
     :param traceback: boolean, if true, print the traceback when an exception occurs.
     """
-    import requests
-
     from aiida_pseudo.groups.family import PseudoDojoFamily
     from .utils import attempt
 
     label = PseudoDojoFamily.format_configuration_label(configuration)
     url_archive = PseudoDojoFamily.get_url_archive(label)
     url_metadata = PseudoDojoFamily.get_url_metadata(label)
 
-    with attempt('downloading selected pseudo potentials archive... ', include_traceback=traceback):
+    with attempt('downloading selected pseudopotentials archive... ', include_traceback=traceback):
         response = requests.get(url_archive)
         response.raise_for_status()
         with open(filepath_archive, 'wb') as handle:
             handle.write(response.content)
             handle.flush()
 
-    with attempt('downloading selected pseudo potentials metadata archive... ', include_traceback=traceback):
+    with attempt('downloading selected pseudopotentials metadata archive... ', include_traceback=traceback):
         response = requests.get(url_metadata)
         response.raise_for_status()
         with open(filepath_metadata, 'wb') as handle:
@@ -212,7 +235,7 @@ def cmd_install_sssp(version, functional, protocol, download_only, traceback):
         family.description = description
         family.set_cutoffs(cutoffs, 'normal', unit='Ry')
 
-        echo.echo_success(f'installed `{label}` containing {family.count()} pseudo potentials')
+        echo.echo_success(f'installed `{label}` containing {family.count()} pseudopotentials')
 
 
 @cmd_install.command('pseudo-dojo')
@@ -335,4 +358,4 @@ def cmd_install_pseudo_dojo(
             family.set_cutoffs(cutoff_values, stringency, unit='Eh')
         family.set_default_stringency(default_stringency)
 
-        echo.echo_success(f'installed `{label}` containing {family.count()} pseudo potentials')
+        echo.echo_success(f'installed `{label}` containing {family.count()} pseudopotentials')

aiida_pseudo/cli/params/options.py

@@ -5,7 +5,7 @@
 import click
 
 from aiida.cmdline.params.options import OverridableOption
-from .types import PseudoPotentialFamilyTypeParam
+from .types import PseudoPotentialFamilyTypeParam, PseudoPotentialTypeParam
 
 __all__ = (
     'VERSION', 'FUNCTIONAL', 'RELATIVISTIC', 'PROTOCOL', 'PSEUDO_FORMAT', 'STRINGENCY', 'DEFAULT_STRINGENCY',
@@ -61,16 +61,28 @@
 )
 
 FAMILY_TYPE = OverridableOption(
-    '-T',
+    '-F',
     '--family-type',
     type=PseudoPotentialFamilyTypeParam(),
     default='pseudo.family',
     show_default=True,
     help='Choose the type of pseudo potential family to create.'
 )
 
+PSEUDO_TYPE = OverridableOption(
+    '-P',
+    '--pseudo-type',
+    type=PseudoPotentialTypeParam(),
+    default='pseudo',
+    show_default=True,
+    help=(
+        'Select the pseudopotential type to be used for the family. Should be the entry point name of a '
+        'subclass of `PseudoPotentialData`.'
+    )
+)
+
 ARCHIVE_FORMAT = OverridableOption(
-    '-F', '--archive-format', type=click.Choice([fmt[0] for fmt in shutil.get_archive_formats()])
+    '-f', '--archive-format', type=click.Choice([fmt[0] for fmt in shutil.get_archive_formats()])
 )
 
 UNIT = OverridableOption(

aiida_pseudo/cli/params/types.py

@@ -1,11 +1,54 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=no-self-use
 """Custom parameter types for command line interface commands."""
+import pathlib
+import typing
+
 import click
+import requests
 
 from aiida.cmdline.params.types import GroupParamType
+from ..utils import attempt
+
+__all__ = ('PseudoPotentialFamilyTypeParam', 'PseudoPotentialFamilyParam', 'PseudoPotentialTypeParam')
+
+
+class PseudoPotentialTypeParam(click.ParamType):
+    """Parameter type for `click` commands to define a subclass of `PseudoPotentialData`."""
+
+    name = 'pseudo_type'
+
+    def convert(self, value, _, __):
+        """Convert the entry point name to the corresponding class.
+
+        :param value: entry point name that should correspond to subclass of `PseudoPotentialData` data plugin
+        :return: the `PseudoPotentialData` subclass
+        :raises: `click.BadParameter` if the entry point cannot be loaded or is not subclass of `PseudoPotentialData`
+        """
+        from aiida.common import exceptions
+        from aiida.plugins import DataFactory
+        from aiida_pseudo.data.pseudo import PseudoPotentialData
+
+        try:
+            pseudo_type = DataFactory(value)
+        except exceptions.EntryPointError as exception:
+            raise click.BadParameter(f'`{value}` is not an existing data plugin.') from exception
 
-__all__ = ('PseudoPotentialFamilyTypeParam', 'PseudoPotentialFamilyParam')
+        if not issubclass(pseudo_type, PseudoPotentialData):
+            raise click.BadParameter(f'`{value}` entry point is not a subclass of `PseudoPotentialData`.')
+
+        PseudoPotentialData.entry_point = value
+
+        return pseudo_type
+
+    def complete(self, _, incomplete):
+        """Return possible completions based on an incomplete value.
+
+        :returns: list of tuples of valid entry points (matching incomplete) and a description
+        """
+        from aiida.plugins.entry_point import get_entry_point_names
+        entry_points = get_entry_point_names('aiida.data')
+        return [(ep, '') for ep in entry_points if (ep.startswith('pseudo') and ep.startswith(incomplete))]
 
 
 class PseudoPotentialFamilyParam(GroupParamType):
@@ -19,6 +62,14 @@ class PseudoPotentialFamilyTypeParam(click.ParamType):
 
     name = 'pseudo_family_type'
 
+    def __init__(self, exclude: typing.Optional[typing.List[str]] = None, **kwargs):
+        """Construct the parameter.
+
+        :param exclude: an optional list of values that should be considered invalid and will raise ``BadParameter``.
+        """
+        super().__init__(**kwargs)
+        self.exclude = exclude
+
     def convert(self, value, _, __):
         """Convert the entry point name to the corresponding class.
 
@@ -35,6 +86,9 @@ def convert(self, value, _, __):
         except exceptions.EntryPointError as exception:
             raise click.BadParameter(f'`{value}` is not an existing group plugin.') from exception
 
+        if self.exclude and value in self.exclude:
+            raise click.BadParameter(f'`{value}` is not an accepted value for this option.')
+
         if not issubclass(family_type, PseudoPotentialFamily):
             raise click.BadParameter(f'`{value}` entry point is not a subclass of `PseudoPotentialFamily`.')
 
@@ -50,3 +104,27 @@ def complete(self, _, incomplete):
         from aiida.plugins.entry_point import get_entry_point_names
         entry_points = get_entry_point_names('aiida.groups')
         return [(ep, '') for ep in entry_points if (ep.startswith('pseudo.family') and ep.startswith(incomplete))]
+
+
+class PathOrUrl(click.Path):
+    """Extension of ``click``'s ``Path``-type that also supports URLs."""
+
+    name = 'PathOrUrl'
+
+    def convert(self, value, param, ctx) -> typing.Union[pathlib.Path, bytes]:
+        """Convert the string value to the desired value.
+
+        If the ``value`` corresponds to a valid path on the local filesystem, return it as a ``pathlib.Path`` instance.
+        Otherwise, treat it as a URL and try to fetch the content. If successful, the raw retrieved bytes will be
+        returned.
+
+        :param value: the filepath on the local filesystem or a URL.
+        """
+        try:
+            # Call the method of the super class, which will raise if it ``value`` is not a valid path.
+            return pathlib.Path(super().convert(value, param, ctx))
+        except click.exceptions.BadParameter:
+            with attempt(f'attempting to download data from `{value}`...'):
+                response = requests.get(value)
+                response.raise_for_status()
+                return response