Merge pull request #363 from querti/errata-item-add-product

Add product name to ContainerImagePushItems

requirements.in

@@ -8,3 +8,4 @@ frozenlist2
 python-dateutil
 kobo
 pytz; python_version < '3.9'
+requests

requirements.txt

@@ -266,6 +266,7 @@ requests==2.31.0 \
  --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
  --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
  # via
+ # -r requirements.in
  # koji
  # requests-gssapi
 requests-gssapi==1.2.3 \

src/pushsource/_impl/backend/errata_source/errata_http_client.py

@@ -0,0 +1,143 @@
+import os
+import threading
+import subprocess
+import re
+import logging
+import tempfile
+from urllib.parse import urljoin
+
+import requests
+import gssapi
+import requests_gssapi
+
+LOG = logging.getLogger("pushsource.errata_http_client")
+
+
+class ErrataHTTPClient:
+ """Class for performing HTTP API queries with Errata."""
+
+ def __init__(self, hostname: str, keytab_path: str = None, principal: str = None):
+ """
+ Initialize.
+
+ Args:
+ hostname (str):
+ Errata hostname.
+ keytab_path (str):
+ Path to a keytab used to perform Kerberos authentication with Errata.
+ If not set, env variable PUSHSOURCE_ERRATA_KEYTAB_PATH will be used.
+ principal (str):
+ Kerberos principal.
+ If not set, env variable PUSHSOURCE_ERRATA_PRINCIPAL will be used.
+ """
+ self.hostname = hostname
+ # Generate a random filename for ccache
+ with tempfile.NamedTemporaryFile(
+ prefix="ccache_pushsource_errata_", delete=False
+ ) as file:
+ self.ccache_filename = file.name
+
+ if keytab_path:
+ self.keytab_path = keytab_path
+ else:
+ self.keytab_path = os.environ.get("PUSHSOURCE_ERRATA_KEYTAB_PATH", None)
+
+ if principal:
+ self.principal = principal
+ else:
+ self.principal = os.environ.get("PUSHSOURCE_ERRATA_PRINCIPAL", None)
+
+ # requests Sessions will be local per thread
+ self._thread_local = threading.local()
+
+ def create_kerberos_ticket(self):
+ """
+ Use the keytab to create a Kerberos ticket granting ticket.
+
+ This method is expected to be called before any HTTP queries to errata are made.
+ """
+ if not self.keytab_path or not self.principal:
+ LOG.warning(
+ "Errata principal or keytab path is not specified. Skipping creating TGT"
+ )
+ return
+
+ result = subprocess.run(
+ ["klist", "-c", f"FILE:{self.ccache_filename}"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ check=False,
+ )
+ regex_res = re.search(r"Default principal: (.*)\n", result.stdout)
+
+ # if Kerberos ticket is not found, or the principal is incorrect
+ if result.returncode or not regex_res or regex_res.group(1) != self.principal:
+ LOG.info(
+ "Errata TGT doesn't exist, running kinit for principal %s",
+ self.principal,
+ )
+ result = subprocess.run(
+ [
+ "kinit",
+ self.principal,
+ "-k",
+ "-t",
+ self.keytab_path,
+ "-c",
+ f"FILE:{self.ccache_filename}",
+ ],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ check=False,
+ )
+ if result.returncode:
+ LOG.warning("kinit has failed: '%s'", result.stdout)
+
+ @property
+ def session(self) -> requests.Session:
+ """
+ Create requests Session.
+
+ Session is used so that Kerberos authentication only has to be done once.
+ As pushsource utilizes threading, and sessions are not thread safe, each thread
+ will have a separate session.
+
+ Returns (object):
+ Authenticated requests Session object.
+ """
+
+ if not hasattr(self._thread_local, "session"):
+ LOG.debug("Creating Errata requests session")
+ name = gssapi.Name(self.principal, gssapi.NameType.user)
+ creds = gssapi.Credentials.acquire(
+ name=name,
+ usage="initiate",
+ store={"ccache": f"FILE:{self.ccache_filename}"},
+ ).creds
+
+ session = requests.Session()
+ session.auth = requests_gssapi.HTTPSPNEGOAuth(creds=creds)
+ self._thread_local.session = session
+
+ return self._thread_local.session
+
+ def get_advisory_data(self, advisory: str) -> dict:
+ """
+ Get advisory data.
+
+ Uses endpoint GET /api/v1/erratum/{id}.
+
+ Args:
+ advisory (str):
+ Advisory ID or name.
+ Returns (dict):
+ Parsed JSON data as returned by Errata.
+ """
+ url = urljoin(self.hostname, f"/api/v1/erratum/{advisory}")
+ LOG.info("Queried Errata HTTP API for %s", advisory)
+ response = self.session.get(url)
+ response.raise_for_status()
+
+ return response.json()

src/pushsource/_impl/backend/errata_source/errata_source.py

@@ -6,6 +6,7 @@
 from more_executors import Executors
 
 from .errata_client import ErrataClient
+from .errata_http_client import ErrataHTTPClient
 
 from ... import compat_attr as attr
 from ...source import Source
@@ -37,6 +38,8 @@ def __init__(
  koji_source=None,
  rpm_filter_arch=None,
  legacy_container_repos=False,
+ keytab_path=None,
+ principal=None,
  threads=4,
  timeout=60 * 60 * 4,
  ):
@@ -68,6 +71,12 @@ def __init__(
  removed when no longer needed. Only use this if you know that you
  need it.
 
+ keytab_path (str):
+ Kerberos keytab path for authenticating with Errata HTTP API.
+
+ principal (str):
+ Kerberos principal for authenticating with Errata HTTP API.
+
  threads (int)
  Number of threads used for concurrent queries to Errata Tool
  and koji.
@@ -79,6 +88,9 @@ def __init__(
  self._url = force_https(url)
  self._errata = list_argument(errata)
  self._client = ErrataClient(threads=threads, url=self._errata_service_url)
+ self._http_client = ErrataHTTPClient(self._url, keytab_path, principal)
+ # Get TGT only once
+ self._http_client.create_kerberos_ticket()
 
  self._rpm_filter_arch = list_argument(rpm_filter_arch, retain_none=True)
 
@@ -225,6 +237,12 @@ def _push_items_from_container_manifests(self, erratum, docker_file_list):
  # }
  #
 
+ # Get product name from Errata. Enrich Container push items with this info
+ advisory_data = self._http_client.get_advisory_data(erratum.name)
+ # This dictionary key is different based on erratum type
+ erratum_type = list(advisory_data["errata"].keys())[0]
+ product_name = advisory_data["errata"][erratum_type]["product"]["name"]
+
  # We'll be getting container metadata from these builds.
  with self._koji_source(
  container_build=list(docker_file_list.keys())
@@ -234,7 +252,7 @@ def _push_items_from_container_manifests(self, erratum, docker_file_list):
  for item in koji_source:
  if isinstance(item, ContainerImagePushItem):
  item = self._enrich_container_push_item(
- erratum, docker_file_list, item
+ erratum, docker_file_list, item, product_name
  )
  elif isinstance(item, OperatorManifestPushItem):
  # Accept this item but nothing special to do
@@ -267,7 +285,9 @@ def _push_items_from_appliance_image_list(self, erratum, appliance_image_list):
 
  return out
 
- def _enrich_container_push_item(self, erratum, docker_file_list, item):
+ def _enrich_container_push_item(
+ self, erratum, docker_file_list, item, product_name
+ ):
  # metadata from koji doesn't contain info about where the image should be
  # pushed and a few other things - enrich it now
  errata_meta = docker_file_list.get(item.build) or {}
@@ -313,7 +333,12 @@ def _enrich_container_push_item(self, erratum, docker_file_list, item):
 
  # koji source provided basic info on container image, ET provides policy on
  # where/how it should be pushed, combine them both to get final push item
- return attr.evolve(item, dest=dest, dest_signing_key=dest_signing_key)
+ return attr.evolve(
+ item,
+ dest=dest,
+ dest_signing_key=dest_signing_key,
+ product_name=product_name,
+ )
 
  def _push_items_from_rpms(self, erratum, rpm_list):
  out = []

src/pushsource/_impl/backend/registry_source.py

@@ -30,7 +30,7 @@
 class RegistrySource(Source):
  """Uses a container image registry as a source of push items."""
 
- def __init__(self, image, dest=None, dest_signing_key=None):
+ def __init__(self, image, dest=None, dest_signing_key=None, product_name=None):
  """Create a new source.
 
  Parameters:
@@ -56,6 +56,10 @@ def __init__(self, image, dest=None, dest_signing_key=None):
  argument can affect the number of generated push items. For example,
  providing two keys would produce double the amount of push items as providing
  a single key.
+
+ product_name (str)
+ If provided, this value will be used to populate
+ :meth:`~pushsource.ContainerImagePushItem.product_name` on generated push items.
  """
  self._images = ["https://%s" % x for x in list_argument(image)]
  if dest:
@@ -65,6 +69,7 @@ def __init__(self, image, dest=None, dest_signing_key=None):
  self._signing_keys = list_argument(dest_signing_key)
  self._inspected = {}
  self._manifests = {}
+ self._product_name = product_name
 
  def __enter__(self):
  return self
@@ -175,6 +180,7 @@ def _push_item_from_registry_uri(self, uri, signing_key):
  labels=labels,
  arch=arch,
  pull_info=pull_info,
+ product_name=self._product_name,
  )
 
  def __iter__(self):

src/pushsource/_impl/model/container.py

@@ -291,6 +291,14 @@ class ContainerImagePushItem(PushItem):
  )
  """Metadata for pulling this image from a registry."""
 
+ product_name = attr.ib(type=str, default=None)
+ """Name of the product of this image.
+
+ Brew doesn't provide this information, so it may not be set if the push
+ items are only generated from brew. This information will be included in the
+ "container security manifests" (formerly known as SBOMs).
+ """
+
 
 @attr.s()
 class SourceContainerImagePushItem(ContainerImagePushItem):

test-requirements.in

@@ -21,3 +21,4 @@ alabaster
 pidiff
 importlib-resources
 bandit
+pytest-mock

test-requirements.txt

@@ -579,10 +579,15 @@ pytest==7.4.1 \
  # via
  # -r test-requirements.in
  # pytest-cov
+ # pytest-mock
 pytest-cov==4.1.0 \
  --hash=sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6 \
  --hash=sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a
  # via -r test-requirements.in
+pytest-mock==3.11.1 \
+ --hash=sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39 \
+ --hash=sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f
+ # via -r test-requirements.in
 python-dateutil==2.8.2 \
  --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
  --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
@@ -657,6 +662,7 @@ requests==2.31.0 \
  --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
  --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
  # via
+ # -r requirements.in
  # coveralls
  # koji
  # requests-gssapi