Skip depsolving based on config flags [RHELDST-18810]

If a flag "base_pkgs_only: True" is passed via ubi config file:
* depsolving is skipped completely
* additional debug(info|source) pkgs are not guessed

So in the end only pkgs defined in ubi config file are added
to the output set.

tests/test_depsolver.py

@@ -798,3 +798,108 @@ def _prepare_test_data_modular_test(pulp):
  all_requires = set([rpm_non_mod_1_require, rpm_mod_1_require])
 
  return repo, all_requires, all_provides, sorted(expected_output_set)
+
+
+def test_run_with_skipped_depsolving(pulp):
+ """
+ Tests that while using 'base-pkgs-only: True' flag in ubi config file,
+ depsolving for RPMs is not run and only pkgs from config file are exported
+ in output. Also guessing names of debug pkgs is skipped.
+ """
+ rpm_rpm, repo_srpm, expected_output_set = _prepare_test_data_skip_depsolving(pulp)
+
+ whitelist = ["gcc", "jq", "perl-version"]
+ dep_item = DepsolverItem(
+ whitelist=whitelist,
+ blacklist=[],
+ in_pulp_repos=[rpm_rpm],
+ )
+
+ flags = {
+ "base_pkgs_only": True,
+ }
+ with Depsolver([dep_item], [repo_srpm], [], **flags) as depsolver:
+ depsolver.run()
+ # check internal state of depsolver object
+ # with provided flag base_pkgs_only:True we don't store any of provides|requires
+ assert depsolver._provides == set()
+
+ assert depsolver._requires == set()
+
+ assert len(depsolver._unsolved) == 0
+
+ # checking correct rpm and srpm names and its associate source repo id
+ output = [
+ (item.name, item.associate_source_repo_id)
+ for item in depsolver.output_set | depsolver.srpm_output_set
+ ]
+
+ assert sorted(output) == expected_output_set
+
+
+def _prepare_test_data_skip_depsolving(pulp):
+ repo_rpm = create_and_insert_repo(id="test_repo_rpm", pulp=pulp)
+ repo_srpm = create_and_insert_repo(id="test_repo_srpm", pulp=pulp)
+
+ unit_1 = RpmUnit(
+ name="gcc",
+ version="10",
+ release="200",
+ epoch="1",
+ arch="x86_64",
+ provides=[RpmDependency(name="lib.a")],
+ requires=[
+ RpmDependency(name="dep-gcc"),
+ RpmDependency(name="lib.b"),
+ RpmDependency(name="lib.c"),
+ ],
+ sourcerpm="gcc.src.rpm",
+ )
+
+ unit_2 = RpmUnit(
+ name="dep-gcc",
+ version="100",
+ release="200",
+ epoch="1",
+ arch="x86_64",
+ provides=[RpmDependency(name="dep-gcc")],
+ requires=[
+ RpmDependency(name="lib.a"),
+ RpmDependency(name="lib.b"),
+ ],
+ sourcerpm="dep-gcc.src.rpm",
+ )
+ # note: the dependency "/some/script" will be skipped from processing
+
+ unit_1_srpm = RpmUnit(
+ name="gcc",
+ filename="gcc.src.rpm",
+ version="1",
+ release="1",
+ epoch="1",
+ arch="x86_64",
+ provides=[],
+ requires=[],
+ content_type_id="srpm",
+ )
+
+ unit_2_srpm = RpmUnit(
+ name="dep-gcc",
+ filename="dep-gcc.src.rpm",
+ version="1",
+ release="1",
+ epoch="1",
+ arch="x86_64",
+ provides=[],
+ requires=[],
+ content_type_id="srpm",
+ )
+
+ pulp.insert_units(repo_rpm, [unit_1, unit_2])
+ pulp.insert_units(repo_srpm, [unit_1_srpm, unit_2_srpm])
+
+ expected_output_set = [(unit.name, "test_repo_rpm") for unit in [unit_1]] + [
+ (unit.name, "test_repo_srpm") for unit in [unit_1_srpm]
+ ]
+
+ return repo_rpm, repo_srpm, sorted(expected_output_set)

tests/test_tasks.py

@@ -1,4 +1,5 @@
 import json
+from functools import partial
 from unittest import mock
 
 import pytest
@@ -922,3 +923,91 @@ def _setup_repos_missing_config(pulp):
  rhel_source_repo = create_and_insert_repo(
  id="rhel_source_repo", pulp=pulp, content_set="cs_srpm_in"
  )
+
+
+def test_multiple_population_sources_skip_depsolving(pulp):
+ _setup_data_multiple_population_sources(pulp)
+
+ with mock.patch("ubi_manifest.worker.tasks.depsolver.utils.Client") as client:
+ with mock.patch(
+ "ubiconfig.get_loader",
+ return_value=MockLoader(flags={"base_pkgs_only": True}),
+ ):
+ with mock.patch(
+ "ubi_manifest.worker.tasks.depsolve.redis.from_url"
+ ) as mock_redis_from_url:
+ redis = MockedRedis(data={})
+ mock_redis_from_url.return_value = redis
+
+ client.return_value = pulp.client
+ # let run the depsolve task
+ result = depsolve.depsolve_task(["ubi_repo"], "fake-url")
+ # we don't return anything useful, everything is saved in redis
+ assert result is None
+
+ # there should 3 keys stored in redis
+ assert sorted(redis.keys()) == [
+ "ubi_debug_repo",
+ "ubi_repo",
+ "ubi_source_repo",
+ ]
+
+ # load json string stored in redis
+ data = redis.get("ubi_repo")
+ content = sorted(json.loads(data), key=lambda d: d["value"])
+ # binary repo contains only 2 rpms but each unit has different src_repo_id
+ assert len(content) == 2
+
+ unit = content[0]
+ assert unit["src_repo_id"] == "rhel_repo-other-2"
+ assert unit["unit_type"] == "RpmUnit"
+ assert unit["unit_attr"] == "filename"
+ assert unit["value"] == "bind-11.200.x86_64.rpm"
+
+ unit = content[1]
+ assert unit["src_repo_id"] == "rhel_repo-2"
+ assert unit["unit_type"] == "RpmUnit"
+ assert unit["unit_attr"] == "filename"
+ assert unit["value"] == "gcc-11.200.x86_64.rpm"
+
+ # load json string stored in redis
+ data = redis.get("ubi_debug_repo")
+ content = sorted(json.loads(data), key=lambda d: d["value"])
+
+ # debuginfo repo is empty because by using flag "base_pkgs_only": True we don't allow
+ # adding additional debug pkgs by guessing their names, but only we allow pkgs defined
+ # in config
+ assert len(content) == 0
+
+ # load json string stored in redis
+ data = redis.get("ubi_source_repo")
+ content = sorted(json.loads(data), key=lambda d: d["value"])
+ # source repo contain 1 SRPM package, correct src_repo_ids
+ # SRPM for gcc packge is not available
+ assert len(content) == 1
+ unit = content[0]
+ assert unit["src_repo_id"] == "rhel_source_repo-other"
+ assert unit["unit_type"] == "RpmUnit"
+ assert unit["unit_attr"] == "filename"
+ assert unit["value"] == "bind_src-2-0.src.rpm"
+
+
+@pytest.mark.parametrize(
+ "flags, consistent",
+ [
+ ({("repo_1", "cs"): {"flag": True}, ("repo_2", "cs"): {"flag": False}}, False),
+ (
+ {("repo_1", "cs"): {"flag": True}, ("repo_2", "cs"): {"other_flag": "foo"}},
+ False,
+ ),
+ ({("repo_1", "cs"): {"flag": True}, ("repo_2", "cs"): {}}, False),
+ ({("repo_1", "cs"): {"flag": True}, ("repo_2", "cs"): {"flag": True}}, True),
+ ],
+)
+def test_validate_depsolver_flags(flags, consistent):
+ _test_call = partial(depsolve.validate_depsolver_flags, flags)
+ if consistent:
+ _test_call() # no exception raised
+ else:
+ with pytest.raises(depsolve.IncosistentDepsolverConfig):
+ _test_call()

tests/utils.py

@@ -13,6 +13,9 @@ def create_and_insert_repo(**kwargs):
 
 
 class MockLoader:
+ def __init__(self, flags=None):
+ self.flags = {"flags": flags or {}}
+
  def load_all(self):
  config_raw_1 = {
  "modules": {
@@ -34,6 +37,7 @@ def load_all(self):
  },
  "arches": ["x86_64", "src"],
  }
+ config_raw_1.update(self.flags)
 
  config_raw_2 = {
  "modules": {
@@ -61,6 +65,8 @@ def load_all(self):
  },
  "arches": ["x86_64", "src"],
  }
+ config_raw_2.update(self.flags)
+
  return [
  ubiconfig.UbiConfig.load_from_dict(config, file, "8")
  for config, file in [(config_raw_1, "file_1"), (config_raw_2, "file_2")]

ubi_manifest/worker/tasks/depsolve.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import random
 from typing import Dict, List
 from collections import defaultdict
 
@@ -29,6 +30,10 @@ class ContentConfigMissing(Exception):
  pass
 
 
+class IncosistentDepsolverConfig(Exception):
+ pass
+
+
 @app.task
 def depsolve_task(ubi_repo_ids: List[str], content_config_url: str) -> None:
  """
@@ -44,6 +49,8 @@ def depsolve_task(ubi_repo_ids: List[str], content_config_url: str) -> None:
  ubi_config_loader = UbiConfigLoader(content_config_url)
 
  with make_pulp_client(app.conf) as client:
+ depsolver_flags = {} # (input_cs, ubi_repo_id): {"flag_x": "value"}
+
  repos_map = {}
  debug_dep_map = {}
  dep_map = {}
@@ -80,11 +87,14 @@ def depsolve_task(ubi_repo_ids: List[str], content_config_url: str) -> None:
  )
  whitelist, debuginfo_whitelist = _filter_whitelist(config)
  blacklist = parse_blacklist_config(config)
+ depsolver_flags[(repo.id, input_cs)] = config.flags.as_dict()
 
  in_source_rpm_repos.extend(_get_population_sources(client, srpm_repo))
 
  dep_map[(repo.id, input_cs)] = DepsolverItem(
- whitelist, blacklist, input_repos
+ whitelist,
+ blacklist,
+ input_repos,
  )
 
  debug_dep_map[(debuginfo_repo.id, input_cs)] = DepsolverItem(
@@ -99,6 +109,7 @@ def depsolve_task(ubi_repo_ids: List[str], content_config_url: str) -> None:
  modulelist, repo, input_repos
  )
 
+ flags = validate_depsolver_flags(depsolver_flags)
  # run modular depsolver
  _LOG.info(
  "Running MODULEMD depsolver for repos: %s",
@@ -120,10 +131,12 @@ def depsolve_task(ubi_repo_ids: List[str], content_config_url: str) -> None:
  repos_map,
  in_source_rpm_repos,
  modulemd_rpm_deps,
+ flags,
  )
 
  _merge_output_dictionary(out, rpm_out)
- _update_debug_whitelist(client, out, debug_dep_map)
+ if not flags.get("base_pkgs_only"):
+ _update_debug_whitelist(client, out, debug_dep_map)
 
  # run depsolver for debuginfo repo
  _LOG.info(
@@ -135,6 +148,7 @@ def depsolve_task(ubi_repo_ids: List[str], content_config_url: str) -> None:
  repos_map,
  in_source_rpm_repos,
  modulemd_rpm_deps,
+ flags,
  )
  # merge 'out' and 'debuginfo_out' dicts without overwriting any entry
  _merge_output_dictionary(out, debuginfo_out)
@@ -227,8 +241,12 @@ def _get_population_sources(client, repo):
  return [client.get_repository(repo_id) for repo_id in repo.population_sources]
 
 
-def _run_depsolver(depolver_items, repos_map, in_source_rpm_repos, modulemd_deps):
- with Depsolver(depolver_items, in_source_rpm_repos, modulemd_deps) as depsolver:
+def _run_depsolver(
+ depsolver_items, repos_map, in_source_rpm_repos, modulemd_deps, flags
+):
+ with Depsolver(
+ depsolver_items, in_source_rpm_repos, modulemd_deps, **flags
+ ) as depsolver:
  depsolver.run()
  exported = depsolver.export()
  out = remap_keys(repos_map, exported)
@@ -290,3 +308,17 @@ def _get_content_config(ubi_config_loader, input_cs, output_cs, version):
  raise ContentConfigMissing
 
  return out
+
+
+def validate_depsolver_flags(depsolver_flags):
+ """
+ Validate all acquired flags, they have to be consistent for all repositories
+ we are processing in one depsolve task otherwise an exception is raised.
+ """
+ reference_flags = random.choice(list(depsolver_flags.values()))
+
+ for flags in depsolver_flags.values():
+ if flags != reference_flags:
+ raise IncosistentDepsolverConfig
+
+ return reference_flags

ubi_manifest/worker/tasks/depsolver/rpm_depsolver.py

@@ -35,6 +35,7 @@ def __init__(
  repos: List[DepsolverItem],
  srpm_repos,
  modulemd_dependencies: Set[str],
+ **kwargs,
  ) -> None:
  self.repos: List[DepsolverItem] = repos
  self.modulemd_dependencies: Set[str] = modulemd_dependencies
@@ -55,6 +56,7 @@ def __init__(
  self._executor: ThreadPoolExecutor = Executors.thread_pool(
  max_workers=MAX_WORKERS
  )
+ self._base_pkgs_only = kwargs.get("base_pkgs_only") or False
 
  def __enter__(self):
  return self
@@ -230,7 +232,7 @@ def run(self):
  source_rpm_fts.append(ft)
 
  to_resolve = set(self.output_set)
- while True:
+ while True and not self._base_pkgs_only:
  # extract provides and requires
  self.extract_and_resolve(to_resolve)
  # we are finished if _ensolved is empty
@@ -257,12 +259,13 @@ def run(self):
  for srpm in srpm_content.result():
  self.srpm_output_set.add(srpm)
 
- # log warnings if depsolving failed
- deps_not_found = {req.name for req in self._requires} - {
- prov.name for prov in self._provides
- }
- if deps_not_found:
- self._log_warnings(deps_not_found, pulp_repos, merged_blacklist)
+ if not self._base_pkgs_only:
+ # log warnings if depsolving failed
+ deps_not_found = {req.name for req in self._requires} - {
+ prov.name for prov in self._provides
+ }
+ if deps_not_found:
+ self._log_warnings(deps_not_found, pulp_repos, merged_blacklist)
 
  def _batch_size(self):
  if len(self._unsolved) < BATCH_SIZE_RESOLVER:

ubi_manifest/worker/tasks/depsolver/utils.py

@@ -3,7 +3,7 @@
 from collections import defaultdict, deque
 from itertools import chain
 from logging import getLogger
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 from pubtools.pulplib import Client, Criteria, Matcher, RpmDependency
 from rpm import labelCompare as label_compare # pylint: disable=no-name-in-module