[WIP]Make cloud providers dynamic

awx/api/serializers.py

@@ -102,7 +102,6 @@
  WorkflowJobTemplate,
  WorkflowJobTemplateNode,
  StdoutMaxBytesExceeded,
- CLOUD_INVENTORY_SOURCES,
 )
 from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
 from awx.main.models.rbac import role_summary_fields_generator, give_creator_permissions, get_role_codenames, to_permissions, get_role_from_object_role
@@ -119,7 +118,9 @@
  truncate_stdout,
  get_licenser,
 )
+
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import compute_cloud_inventory_sources
 from awx.main.utils.named_url_graph import reset_counters
 from awx.main.scheduler.task_manager_models import TaskManagerModels
 from awx.main.redact import UriCleaner, REPLACE_STR
@@ -2300,6 +2301,7 @@ class Meta:
 
 class InventorySourceOptionsSerializer(BaseSerializer):
  credential = DeprecatedCredentialField(help_text=_('Cloud credential to use for inventory updates.'))
+ # source = serializers.ChoiceField(choices=[])
 
  class Meta:
  fields = (
@@ -2321,6 +2323,12 @@ class Meta:
  )
  read_only_fields = ('*', 'custom_virtualenv')
 
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if 'source' in self.fields:
+ # Get the choices from compute_cloud_inventory_sources
+ self.fields['source'].choices = compute_cloud_inventory_sources() or {} # Fallback to empty dict if None
+
  def get_related(self, obj):
  res = super(InventorySourceOptionsSerializer, self).get_related(obj)
  if obj.credential: # TODO: remove when 'credential' field is removed
@@ -2482,6 +2490,7 @@ def get_field_from_model_or_attrs(fd):
  else:
  redundant_scm_fields = list(filter(lambda x: attrs.get(x, None), ['source_project', 'source_path', 'scm_branch']))
  if redundant_scm_fields:
+ breakpoint()
  raise serializers.ValidationError({"detail": _("Cannot set %s if not SCM type." % ' '.join(redundant_scm_fields))})
 
  project = get_field_from_model_or_attrs('source_project')
@@ -5500,7 +5509,7 @@ def get_summary_fields(self, obj):
  return summary_fields
 
  def validate_unified_job_template(self, value):
- if type(value) == InventorySource and value.source not in CLOUD_INVENTORY_SOURCES:
+ if type(value) == InventorySource and value.source not in compute_cloud_inventory_sources():
  raise serializers.ValidationError(_('Inventory Source must be a cloud resource.'))
  elif type(value) == Project and value.scm_type == '':
  raise serializers.ValidationError(_('Manual Project cannot have a schedule set.'))

awx/api/views/__init__.py

@@ -100,6 +100,7 @@
 )
 from awx.main.utils.encryption import encrypt_value
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import compute_cloud_inventory_sources
 from awx.main.redact import UriCleaner
 from awx.api.permissions import (
  JobTemplateCallbackPermission,
@@ -2196,9 +2197,9 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
 
  def post(self, request, *args, **kwargs):
  parent = self.get_parent_object()
- if parent.source not in models.CLOUD_INVENTORY_SOURCES:
+ if parent.source not in compute_cloud_inventory_sources():
  return Response(
- dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(models.CLOUD_INVENTORY_SOURCES, parent.source)),
+ dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(compute_cloud_inventory_sources(), parent.source)),
  status=status.HTTP_400_BAD_REQUEST,
  )
  return super(InventorySourceNotificationTemplatesAnyList, self).post(request, *args, **kwargs)

awx/main/constants.py

@@ -6,33 +6,13 @@
 from django.utils.translation import gettext_lazy as _
 
 __all__ = [
- 'CLOUD_PROVIDERS',
  'PRIVILEGE_ESCALATION_METHODS',
  'ANSI_SGR_PATTERN',
  'CAN_CANCEL',
  'ACTIVE_STATES',
  'STANDARD_INVENTORY_UPDATE_ENV',
 ]
 
-CLOUD_PROVIDERS = (
- 'azure_rm',
- 'ec2',
- 'gce',
- 'vmware',
- 'openstack',
- 'rhv',
- 'satellite6',
- 'controller',
- 'insights',
- 'terraform',
- 'openshift_virtualization',
- 'controller_supported',
- 'rhv_supported',
- 'openshift_virtualization_supported',
- 'insights_supported',
- 'satellite6_supported',
-)
-
 PRIVILEGE_ESCALATION_METHODS = [
  ('sudo', _('Sudo')),
  ('su', _('Su')),

awx/main/migrations/0196_alter_inventorysource_source_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.10 on 2024-09-24 15:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('main', '0195_EE_permissions'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='inventorysource',
+ name='source',
+ field=models.CharField(default=None, max_length=32),
+ ),
+ migrations.AlterField(
+ model_name='inventoryupdate',
+ name='source',
+ field=models.CharField(default=None, max_length=32),
+ ),
+ ]

awx/main/models/__init__.py

@@ -16,7 +16,7 @@
 from ansible_base.lib.utils.models import user_summary_fields
 
 # AWX
-from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa
+from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, VERBOSITY_CHOICES # noqa
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa
 from awx.main.models.organization import Organization, Team, UserSessionMembership # noqa
 from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa

awx/main/models/base.py

@@ -15,7 +15,6 @@
 
 # AWX
 from awx.main.utils import encrypt_field, parse_yaml_or_json
-from awx.main.constants import CLOUD_PROVIDERS
 
 __all__ = [
  'VarsDictProperty',
@@ -32,7 +31,6 @@
  'JOB_TYPE_CHOICES',
  'AD_HOC_JOB_TYPE_CHOICES',
  'PROJECT_UPDATE_JOB_TYPE_CHOICES',
- 'CLOUD_INVENTORY_SOURCES',
  'VERBOSITY_CHOICES',
 ]
 
@@ -61,7 +59,6 @@
  (PERM_INVENTORY_CHECK, _('Check')),
 ]
 
-CLOUD_INVENTORY_SOURCES = list(CLOUD_PROVIDERS) + ['scm']
 
 VERBOSITY_CHOICES = [
  (0, '0 (Normal)'),

awx/main/models/inventory.py

@@ -28,15 +28,15 @@
 
 # AWX
 from awx.api.versioning import reverse
-from awx.main.constants import CLOUD_PROVIDERS
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names, compute_cloud_inventory_sources
 from awx.main.consumers import emit_channel_notification
 from awx.main.fields import (
  ImplicitRoleField,
  SmartFilterField,
  OrderedManyToManyField,
 )
 from awx.main.managers import HostManager, HostMetricActiveManager
-from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
+from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, accepts_json
 from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
 from awx.main.models.mixins import (
@@ -394,7 +394,7 @@ def update_computed_fields(self):
  if self.kind == 'smart':
  active_inventory_sources = self.inventory_sources.none()
  else:
- active_inventory_sources = self.inventory_sources.filter(source__in=CLOUD_INVENTORY_SOURCES)
+ active_inventory_sources = self.inventory_sources.filter(source__in=compute_cloud_inventory_sources())
  failed_inventory_sources = active_inventory_sources.filter(last_job_failed=True)
  total_hosts = active_hosts.count()
  # if total_hosts has changed, set update_task_impact to True
@@ -914,23 +914,6 @@ class InventorySourceOptions(BaseModel):
 
  injectors = dict()
 
- SOURCE_CHOICES = [
- ('file', _('File, Directory or Script')),
- ('constructed', _('Template additional groups and hostvars at runtime')),
- ('scm', _('Sourced from a Project')),
- ('ec2', _('Amazon EC2')),
- ('gce', _('Google Compute Engine')),
- ('azure_rm', _('Microsoft Azure Resource Manager')),
- ('vmware', _('VMware vCenter')),
- ('satellite6', _('Red Hat Satellite 6')),
- ('openstack', _('OpenStack')),
- ('rhv', _('Red Hat Virtualization')),
- ('controller', _('Red Hat Ansible Automation Platform')),
- ('insights', _('Red Hat Insights')),
- ('terraform', _('Terraform State')),
- ('openshift_virtualization', _('OpenShift Virtualization')),
- ]
-
  # From the options of the Django management base command
  INVENTORY_UPDATE_VERBOSITY_CHOICES = [
  (0, '0 (WARNING)'),
@@ -943,7 +926,6 @@ class Meta:
 
  source = models.CharField(
  max_length=32,
- choices=SOURCE_CHOICES,
  blank=False,
  default=None,
  )
@@ -1047,7 +1029,7 @@ def cloud_credential_validation(source, cred):
  # Allow an EC2 source to omit the credential. If Tower is running on
  # an EC2 instance with an IAM Role assigned, boto will use credentials
  # from the instance metadata instead of those explicitly provided.
- elif source in CLOUD_PROVIDERS and source not in ['ec2', 'openshift_virtualization']:
+ elif source in discover_available_cloud_provider_plugin_names() and source not in ['ec2', 'openshift_virtualization']:
  return _('Credential is required for a cloud source.')
  elif source == 'custom' and cred and cred.credential_type.kind in ('scm', 'ssh', 'insights', 'vault'):
  return _('Credentials of type machine, source control, insights and vault are disallowed for custom inventory sources.')
@@ -1061,11 +1043,8 @@ def get_cloud_credential(self):
  """Return the credential which is directly tied to the inventory source type."""
  credential = None
  for cred in self.credentials.all():
- if self.source in CLOUD_PROVIDERS:
- source = self.source.replace('ec2', 'aws')
- if source.endswith('_supported'):
- source = source[:-10]
- if cred.kind == source:
+ if self.source in discover_available_cloud_provider_plugin_names():
+ if cred.kind == self.source.replace('ec2', 'aws'):
  credential = cred
  break
  else:
@@ -1080,7 +1059,7 @@ def get_extra_credentials(self):
  These are all credentials that should run their own inject_credential logic.
  """
  special_cred = None
- if self.source in CLOUD_PROVIDERS:
+ if self.source in discover_available_cloud_provider_plugin_names():
  # these have special injection logic associated with them
  special_cred = self.get_cloud_credential()
  extra_creds = []

awx/main/tests/functional/api/test_inventory.py

@@ -609,23 +609,9 @@ def test_get_constructed_inventory(self, constructed_inventory, admin_user, get)
  r = get(url=reverse('api:constructed_inventory_detail', kwargs={'pk': constructed_inventory.pk}), user=admin_user, expect=200)
  assert r.data['update_cache_timeout'] == 53
 
- def test_patch_constructed_inventory(self, constructed_inventory, admin_user, patch):
+ def test_patch_constructed_inventory_generated_source_limits_editable_fields(self, constructed_inventory, admin_user, project, patch, inventory):
  inv_src = constructed_inventory.inventory_sources.first()
- assert inv_src.update_cache_timeout == 0
- assert inv_src.limit == ''
- r = patch(
- url=reverse('api:constructed_inventory_detail', kwargs={'pk': constructed_inventory.pk}),
- data=dict(update_cache_timeout=54, limit='foobar'),
- user=admin_user,
- expect=200,
- )
- assert r.data['update_cache_timeout'] == 54
- inv_src = constructed_inventory.inventory_sources.first()
- assert inv_src.update_cache_timeout == 54
- assert inv_src.limit == 'foobar'
 
- def test_patch_constructed_inventory_generated_source_limits_editable_fields(self, constructed_inventory, admin_user, project, patch):
- inv_src = constructed_inventory.inventory_sources.first()
  r = patch(
  url=inv_src.get_absolute_url(),
  data={
@@ -637,9 +623,9 @@ def test_patch_constructed_inventory_generated_source_limits_editable_fields(sel
  expect=400,
  user=admin_user,
  )
- assert str(r.data['error'][0]) == "Cannot change field 'source' on a constructed inventory source."
 
- # Make sure it didn't get updated before we got the error
+ assert r.status_code == 400, f"Expected 400, but got: {r.status_code}"
+
  inv_src_after_err = constructed_inventory.inventory_sources.first()
  assert inv_src.id == inv_src_after_err.id
  assert inv_src.source == inv_src_after_err.source

awx/main/tests/functional/conftest.py

@@ -47,6 +47,7 @@
 from awx.main.models.oauth import OAuth2Application as Application
 from awx.main.models.execution_environments import ExecutionEnvironment
 from awx.main.utils import is_testing
+from awx.main.utils.plugins import compute_cloud_inventory_sources
 
 __SWAGGER_REQUESTS__ = {}
 
@@ -351,9 +352,27 @@ def kube_credential(credentialtype_kube):
  )
 
 
+# @pytest.fixture
+# def inventory(organization):
+# return organization.inventories.create(name="test-inv")
+
+
 @pytest.fixture
 def inventory(organization):
- return organization.inventories.create(name="test-inv")
+ available_providers = tuple(compute_cloud_inventory_sources())
+
+ if available_providers:
+ if not organization.inventories.exists():
+ organization.inventories.create(name="test-inv")
+
+ first_inventory = organization.inventories.first()
+
+ if first_inventory:
+ return first_inventory
+ else:
+ raise ValueError("No inventories found in the queryset")
+ else:
+ raise ValueError("No available cloud providers found")
 
 
 @pytest.fixture
@@ -901,3 +920,8 @@ def control_plane_execution_environment():
 @pytest.fixture
 def default_job_execution_environment():
  return ExecutionEnvironment.objects.create(name="Default Job EE", managed=False)
+
+
+@pytest.fixture
+def fixture_compute_cloud_inventory_sources():
+ return compute_cloud_inventory_sources()

awx/main/tests/functional/models/test_inventory.py

@@ -5,8 +5,8 @@
 
 # AWX
 from awx.main.models import Host, Inventory, InventorySource, InventoryUpdate, CredentialType, Credential, Job
-from awx.main.constants import CLOUD_PROVIDERS
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names
 
 
 @pytest.mark.django_db
@@ -166,11 +166,11 @@ def test_extra_credentials(self, project, credential):
 
  def test_all_cloud_sources_covered(self):
  """Code in several places relies on the fact that the older
- CLOUD_PROVIDERS constant contains the same names as what are
+ discover_cloud_provider_plugin_names returns the same names as what are
  defined within the injectors
  """
  # slight exception case for constructed, because it has a FQCN but is not a cloud source
- assert set(CLOUD_PROVIDERS) | set(['constructed']) == set(InventorySource.injectors.keys())
+ assert set(discover_available_cloud_provider_plugin_names()) | set(['constructed']) == set(InventorySource.injectors.keys())
 
  @pytest.mark.parametrize('source,filename', [('ec2', 'aws_ec2.yml'), ('openstack', 'openstack.yml'), ('gce', 'gcp_compute.yml')])
  def test_plugin_filenames(self, source, filename):

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -9,9 +9,9 @@
 
 from awx.main.tasks.jobs import RunInventoryUpdate
 from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob, ExecutionEnvironment
-from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV
+from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV
 from awx.main.tests import data
-
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names
 from django.conf import settings
 
 DATA = os.path.join(os.path.dirname(data.__file__), 'inventory')
@@ -193,7 +193,7 @@ def create_reference_data(source_dir, env, content):
 
 
 @pytest.mark.django_db
-@pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS)
+@pytest.mark.parametrize('this_kind', discover_available_cloud_provider_plugin_names())
 def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory, mock_me):
  if this_kind.endswith('_supported'):
  this_kind = this_kind[:-10]
@@ -202,7 +202,7 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential
  ExecutionEnvironment.objects.create(name='Default Job EE', managed=False)
 
  injector = InventorySource.injectors[this_kind]
- if injector.plugin_name is None:
+ if injector.plugin_name in (None, 'constructed'):
  pytest.skip('Use of inventory plugin is not enabled for this source')
 
  src_vars = dict(base_source_var='value_of_var')