Add Client.create_repository() [RHELDST-22483]

`Client` is now capable of creating a repository on pulp
server. The repository is initialized with proper `Importer` in order
to enable sync/upload of content to new repositories.

Another notable changes:
* 409 status is not retried in requests as it is used as indicator of
 existing repository.
* Added create_repository() method to FakeClient as well.
* Small changes in the repository.yaml schema as some of the fields
  don't have to be necessarily initialized when creating repo but can be
  updated later.
* Added `Importer` pulp object mapping class and related subclasses
  that encapsulate minimal required fields for this object with valid
  defaults.

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-- n/a
+- Added `Client.create_repository()` method
 
 ## [2.38.1] - 2023-12-14
 

docs/api/files.rst

@@ -14,6 +14,8 @@ Repository
 .. autoclass:: pubtools.pulplib.FileSyncOptions()
  :members:
 
+.. autoclass:: pubtools.pulplib.IsoImporter()
+ :members:
 
 Units
 -----

docs/api/pulpcore.rst

@@ -27,6 +27,8 @@ Repository
 .. autoclass:: pubtools.pulplib.SyncOptions()
  :members:
 
+.. autoclass:: pubtools.pulplib.Importer()
+ :members:
 
 Units
 -----

docs/api/yum.rst

@@ -14,6 +14,8 @@ Repository
 .. autoclass:: pubtools.pulplib.YumSyncOptions()
  :members:
 
+.. autoclass:: pubtools.pulplib.YumImporter()
+ :members:
 
 Units: RPM
 ----------

pubtools/pulplib/__init__.py

@@ -31,5 +31,8 @@
  Task,
  MaintenanceReport,
  MaintenanceEntry,
+ Importer,
+ YumImporter,
+ IsoImporter,
 )
 from ._impl.fake import FakeController

pubtools/pulplib/_impl/client/client.py

@@ -1054,3 +1054,46 @@ def _do_sync(self, repo_id, sync_options):
  return self._task_executor.submit(
  self._do_request, method="POST", url=url, json=body
  )
+
+ def create_repository(self, repo):
+ """Create a repository with initial data provided in the
+ argument. Importer for repository is automatically associated
+ if available. If repository already exists, only warning is logged.
+
+ Args:
+ repo (:class:`~pubtools.pulplib.Repository`)
+ A repository object used for creation.
+
+ Returns:
+ Future
+ A future which is resolved with a value of ``None`` once the
+ repository has been created.
+
+ .. versionadded:: 2.39.0
+ """
+ url = os.path.join(self._url, "pulp/api/v2/repositories/")
+
+ body = repo._to_data()
+ repo_id = body["id"]
+
+ importer = body.pop("importers", [])
+ body["importer_type_id"] = importer[0]["importer_type_id"] if importer else None
+ body["importer_config"] = importer[0]["config"] if importer else None
+
+ def log_existing_repo(exception):
+ if (
+ getattr(exception, "response", None) is not None
+ and exception.response.status_code == 409
+ ):
+ LOG.warning("Repository %s already exists", repo_id)
+ return None
+
+ raise exception
+
+ LOG.debug("Creating repository %s", repo_id)
+ out = self._request_executor.submit(
+ self._do_request, method="POST", url=url, json=body
+ )
+
+ out = f_map(out, error_fn=log_existing_repo)
+ return f_map(out, lambda _: None)

pubtools/pulplib/_impl/client/retry.py

@@ -29,8 +29,8 @@ def should_retry(self, attempt, future):
 
  exception = future.exception()
  if exception and getattr(exception, "response", None) is not None:
- # if returned status code is 404, never retry on that
- if exception.response.status_code == 404:
+ # if returned status code is 404 or 409, never retry on that
+ if exception.response.status_code in (404, 409):
  return False
 
  if exception and retry:

pubtools/pulplib/_impl/fake/client.py

@@ -651,3 +651,12 @@ def _do_sync(self, repo_id, sync_config): # pylint:disable = unused-argument
  self._state.sync_history.append(Sync(repo_f.result(), [task], sync_config))
 
  return f_return([task])
+
+ def create_repository(self, repo):
+ with self._state.lock:
+ if repo.id not in [
+ existing_repo.id for existing_repo in self._state.repositories
+ ]:
+ self._state.repositories.append(repo)
+
+ return f_return(None)

pubtools/pulplib/_impl/model/__init__.py

@@ -9,6 +9,9 @@
  FileSyncOptions,
  ContainerSyncOptions,
  YumSyncOptions,
+ Importer,
+ IsoImporter,
+ YumImporter,
 )
 from .unit import (
  Unit,

pubtools/pulplib/_impl/model/common.py

@@ -234,3 +234,15 @@ def _delete(self, resource_type, resource_id):
  delete_f = client._delete_resource(resource_type, resource_id)
  delete_f = f_map(delete_f, self.__detach)
  return f_proxy(delete_f)
+
+
+def schemaless_init(cls, data):
+ # Construct and return an instance of (attrs-using) cls from
+ # pulp data, where data in pulp has no schema at all (and hence
+ # every field could possibly be missing).
+ kwargs = {}
+ for key in [fld.name for fld in attr.fields(cls)]:
+ if key in data:
+ kwargs[key] = data[key]
+
+ return cls(**kwargs)

pubtools/pulplib/_impl/model/repository/__init__.py

@@ -1,4 +1,4 @@
-from .base import Repository, PublishOptions, SyncOptions
+from .base import Repository, PublishOptions, SyncOptions, Importer
 from .container import ContainerImageRepository, ContainerSyncOptions
-from .yum import YumRepository, YumSyncOptions
-from .file import FileRepository, FileSyncOptions
+from .yum import YumRepository, YumSyncOptions, YumImporter
+from .file import FileRepository, FileSyncOptions, IsoImporter

pubtools/pulplib/_impl/model/repository/base.py

@@ -19,6 +19,7 @@
 from ...hooks import pm
 from ...util import dict_put, lookup, ABSENT
 
+from ..common import schemaless_init
 
 LOG = logging.getLogger("pubtools.pulplib")
 
@@ -35,6 +36,39 @@ def decorate(klass):
  return decorate
 
 
+@attr.s(kw_only=True, frozen=True)
+class Importer(PulpObject):
+ """
+ Importer is a pulp object that needs to be associated with repository
+ in order to successfully sync or upload content to it.
+ """
+
+ type_id = pulp_attrib(default=None, type=str, pulp_field="importer_type_id")
+ """
+ Type id of the importer.
+ """
+ config = pulp_attrib(default=None, type=dict, pulp_field="config")
+ """
+ Configuration dictionary of the importer.
+ """
+
+ @classmethod
+ def _from_data(cls, data):
+ # Convert from raw list/dict as provided in Pulp responses into model.
+ if isinstance(data, list):
+ return cls._from_data(data[0]) if data else schemaless_init(cls, data)
+
+ return schemaless_init(cls, data)
+
+ def _to_data(self):
+ return [
+ {
+ "importer_type_id": self.type_id,
+ "config": self.config,
+ }
+ ]
+
+
 @attr.s(kw_only=True, frozen=True)
 class PublishOptions(object):
  """Options controlling a repository
@@ -339,6 +373,19 @@ class Repository(PulpObject, Deletable):
  .. versionadded:: 2.37.0
  """
 
+ importer = pulp_attrib(
+ default=Importer(),
+ type=Importer,
+ pulp_field="importers",
+ pulp_py_converter=Importer._from_data,
+ py_pulp_converter=Importer._to_data,
+ )
+ """
+ An object of :class:`~pubtools.pulplib.Importer` that is associated with the repository.
+
+ .. versionadded:: 2.39.0 
+ """
+
  @distributors.validator
  def _check_repo_id(self, _, value):
  # checks if distributor's repository id is same as the repository it

pubtools/pulplib/_impl/model/repository/file.py

@@ -4,7 +4,7 @@
 from attr import validators
 from frozenlist2 import frozenlist
 
-from .base import Repository, SyncOptions, repo_type
+from .base import Repository, SyncOptions, repo_type, Importer
 from ...model.unit import FileUnit
 from ..attr import pulp_attrib
 from ... import compat_attr as attr
@@ -13,6 +13,16 @@
 LOG = logging.getLogger("pubtools.pulplib")
 
 
+@attr.s(kw_only=True, frozen=True)
+class IsoImporter(Importer):
+ type_id = pulp_attrib(
+ default="iso_importer", type=str, pulp_field="importer_type_id"
+ )
+ """
+ Specific importer_type_id for File repositories.
+ """
+
+
 @attr.s(kw_only=True, frozen=True)
 class FileSyncOptions(SyncOptions):
  """Options controlling a file repository
@@ -49,6 +59,19 @@ class FileRepository(Repository):
  converter=frozenlist,
  )
 
+ importer = pulp_attrib(
+ default=IsoImporter(),
+ type=IsoImporter,
+ pulp_field="importers",
+ pulp_py_converter=IsoImporter._from_data,
+ py_pulp_converter=IsoImporter._to_data,
+ )
+ """
+ An object of :class:`~pubtools.pulplib.IsoImporter` that is associated with the repository.
+
+ .. versionadded:: 2.39.0 
+ """
+
  def upload_file(self, file_obj, relative_url=None, **kwargs):
  """Upload a file to this repository.
 

pubtools/pulplib/_impl/model/repository/yum.py

@@ -3,14 +3,24 @@
 from frozenlist2 import frozenlist
 
 from more_executors.futures import f_map, f_proxy, f_return, f_zip, f_flat_map
-from .base import Repository, SyncOptions, repo_type
+from .base import Repository, SyncOptions, repo_type, Importer
 from ..attr import pulp_attrib
 from ..common import DetachedException
 from ...model.unit import RpmUnit
 from ... import compat_attr as attr, comps
 from ...criteria import Criteria, Matcher
 
 
+@attr.s(kw_only=True, frozen=True)
+class YumImporter(Importer):
+ type_id = pulp_attrib(
+ default="yum_importer", type=str, pulp_field="importer_type_id"
+ )
+ """
+ Specific importer_type_id for Yum repositories.
+ """
+
+
 @attr.s(kw_only=True, frozen=True)
 class YumSyncOptions(SyncOptions):
  """Options controlling a yum repository
@@ -103,6 +113,19 @@ class YumRepository(Repository):
  )
  """Version of UBI config that should be used for population of this repository."""
 
+ importer = pulp_attrib(
+ default=YumImporter(),
+ type=YumImporter,
+ pulp_field="importers",
+ pulp_py_converter=YumImporter._from_data,
+ py_pulp_converter=YumImporter._to_data,
+ )
+ """
+ An object of :class:`~pubtools.pulplib.YumImporter` that is associated with the repository.
+
+ .. versionadded:: 2.39.0 
+ """
+
  def get_binary_repository(self):
  """Find and return the binary repository relating to this repository.
 

pubtools/pulplib/_impl/model/unit/base.py

@@ -139,15 +139,3 @@ def _usermeta_from_kwargs(cls, **kwargs):
  )
 
  return out
-
-
-def schemaless_init(cls, data):
- # Construct and return an instance of (attrs-using) cls from
- # pulp data, where data in pulp has no schema at all (and hence
- # every field could possibly be missing).
- kwargs = {}
- for key in [fld.name for fld in attr.fields(cls)]:
- if key in data:
- kwargs[key] = data[key]
-
- return cls(**kwargs)

pubtools/pulplib/_impl/model/unit/erratum.py

@@ -1,8 +1,9 @@
 from frozenlist2 import frozenlist
 
-from .base import Unit, PulpObject, unit_type, schemaless_init
+from .base import Unit, PulpObject, unit_type
 
 from ..attr import pulp_attrib
+from ..common import schemaless_init
 from ... import compat_attr as attr
 from ..validate import (
  optional_bool,

pubtools/pulplib/_impl/model/unit/modulemd.py

@@ -1,8 +1,9 @@
 import re
 
-from .base import Unit, unit_type, schemaless_init
+from .base import Unit, unit_type
 
 from ..attr import pulp_attrib
+from ..common import schemaless_init
 from ... import compat_attr as attr
 from ..convert import (
  frozenlist_or_none_converter,

pubtools/pulplib/_impl/model/unit/rpm.py

@@ -1,9 +1,10 @@
 import datetime
 
 from pubtools.pulplib._impl.model.validate import optional_list_of
-from .base import Unit, unit_type, schemaless_init
+from .base import Unit, unit_type
 
 from ..attr import pulp_attrib
+from ..common import schemaless_init
 from ... import compat_attr as attr
 from ..convert import (
  frozenlist_or_none_converter,

pubtools/pulplib/_impl/schema/repository.yaml

@@ -69,7 +69,9 @@ properties:
  # Note that this field is not set by Pulp itself. Only certain tools
  # are expected to initialize this field when creating a repo.
  created:
- type: string
+ anyOf:
+ - type: "null"
+ - type: string
  # example: 2019-06-05T11:56:50Z
  pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z"
 
@@ -100,7 +102,9 @@ properties:
 
  # Version of ubi config that should be used for population of this repository
  ubi_config_version:
- type: string
+ anyOf:
+ - type: "null"
+ - type: string
 
  # Flag indicating whether the repository is visible in production instance
  # of download service. Stored as string.

setup.py

@@ -21,7 +21,7 @@ def get_requirements():
 
 setup(
  name="pubtools-pulplib",
- version="2.38.1",
+ version="2.39.0",
  packages=find_packages(exclude=["tests"]),
  package_data={"pubtools.pulplib._impl.schema": ["*.yaml"]},
  url="https://github.com/release-engineering/pubtools-pulplib",