Merge pull request #60 from release-engineering/switch-async-publish

Returns asyncio to publish in push.

src/pubtools/_marketplacesvm/cloud_providers/aws.py

@@ -451,8 +451,6 @@ def _pre_publish(self, push_item: AmiPushItem, **kwargs) -> Tuple[AmiPushItem, A
  Tuple[AmiPushItem, Any]
  The incoming push item and the dict with received parameters.
  """
- LOG.info("Checking for active changesets in: %s", push_item.dest[0])
- self.publish_svc.wait_active_changesets(push_item.dest[0])
  return push_item, kwargs
 
  def _publish(
@@ -474,6 +472,10 @@ def _publish(
  Whether to replace every image in the product with the given one or not.
  Defaults to ``False``
  """
+ # Check if this product is locked currently and wait for it to become unlocked
+ LOG.info("Checking for active changesets in: %s", push_item.dest[0])
+ self.publish_svc.wait_active_changesets(push_item.dest[0])
+
  if push_item.release.base_product is not None:
  os_name = push_item.release.base_product
  else:

src/pubtools/_marketplacesvm/tasks/push/command.py

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
+import asyncio
 import datetime
 import json
 import logging
@@ -256,7 +257,7 @@ def _push_pre_publish(self, upload_result: List[UPLOAD_RESULT]) -> List[UPLOAD_R
  res.append((mapped_item, starmap_query))
  return res
 
- def _push_publish(self, upload_result: List[UPLOAD_RESULT]) -> List[Dict[str, Any]]:
+ async def _push_publish(self, upload_result: List[UPLOAD_RESULT]) -> List[Dict[str, Any]]:
  """
  Perform the publishing for the the VM images.
 
@@ -267,25 +268,12 @@ def _push_publish(self, upload_result: List[UPLOAD_RESULT]) -> List[Dict[str, An
  Dictionary with the resulting operation for the Collector service.
  """
 
- def push_function(mapped_item, marketplace, starmap_query) -> Dict[str, Any]:
+ async def push_function(mapped_item, marketplace, starmap_query) -> Dict[str, Any]:
  # Get the push item for the current marketplace
  pi = mapped_item.get_push_item_for_marketplace(marketplace)
 
  # Associate image with Product/Offer/Plan and publish only if it's not a pre-push
  if pi.state != State.UPLOADFAILED and not self.args.pre_push:
- # The first publish should always be with `pre_push` set True because it might
- # happen that one offer with multiple plans would receive the same image and
- # we can't `publish` the offer with just the first plan changed and try to change
- # the others (every plan should be changed while the offer is still on draft).
- #
- # Then this first `_publish` call is intended to only associate the image with
- # all the offers/plans but not change it to live, when this is applicable.
- pi = self._publish(marketplace, pi)
-
- # Once we associated all the images with their offer/plans it's now safe to call
- # again the publish if and only if `pre_push == False`.
- # The indepondent operation will guarantee that the images are already associated
- # with the Product/Offer/Plan and just the go-live part is called.
  pi = self._publish(
  marketplace,
  pi,
@@ -311,23 +299,17 @@ def push_function(mapped_item, marketplace, starmap_query) -> Dict[str, Any]:
 
  res_output = []
 
- # Sequentially publish the uploaded items for each marketplace.
- # It's recommended to do this operation sequentially since parallel publishing in the
- # same marketplace may cause errors due to the change set already being applied.
- for mapped_item, starmap_query in upload_result:
- to_await = []
- executor = Executors.thread_pool(
- name="pubtools-marketplacesvm-push-regions",
- max_workers=min(max(len(mapped_item.marketplaces), 1), self._PROCESS_THREADS),
- )
-
- for marketplace in mapped_item.marketplaces:
- to_await.append(
- executor.submit(push_function, mapped_item, marketplace, starmap_query)
+ # This publishes in parallel since there shouldn't be collision between the
+ # differing marketplaces.
+ res_output.extend(
+ await asyncio.gather(
+ *(
+ push_function(mapped_item, marketplace, starmap_query)
+ for mapped_item, starmap_query in upload_result
+ for marketplace in mapped_item.marketplaces
  )
-
- for f_out in to_await:
- res_output.append(f_out.result())
+ )
+ )
 
  return res_output
 
@@ -422,7 +404,7 @@ def run(self, collect_results: bool = True, allow_empty_targets: bool = False) -
  upload_result = self._push_pre_publish(upload_result)
 
  # 4 - Publish the uploaded images letting the external function to control the threads
- result = self._push_publish(upload_result)
+ result = asyncio.run(self._push_publish(upload_result))
 
  # process result for failures
  failed = False

tests/cloud_providers/test_provider_aws.py

@@ -611,9 +611,6 @@ def test_upload_custom_tags(
 
 def test_pre_publish(aws_push_item: AmiPushItem, fake_aws_provider: AWSProvider):
  pi, res = fake_aws_provider._pre_publish(aws_push_item, param=True)
- fake_aws_provider.publish_svc.wait_active_changesets.assert_called_once_with(
- aws_push_item.dest[0]
- )
  assert (pi, res) == (aws_push_item, {"param": True})
 
 
@@ -686,6 +683,9 @@ def test_publish(
  mock_metadata.assert_called_once_with(**metadata)
  fake_aws_provider.publish_svc.publish.assert_called_once_with(meta_obj)
  fake_aws_provider.upload_svc_partial.upload.assert_not_called() # type: ignore [attr-defined] # noqa: E501
+ fake_aws_provider.publish_svc.wait_active_changesets.assert_called_once_with(
+ aws_push_item.dest[0]
+ )
 
 
 @pytest.mark.parametrize("new_base_product", ["test-base", None])

tests/logs/combined_push/test_combined_push/test_do_advisory_push.txt

@@ -36,9 +36,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Marketplace VM push completed
 [ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build,ami_build2

tests/logs/combined_push/test_combined_push/test_do_allowed_empty_mapping_push.txt

@@ -36,9 +36,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Marketplace VM push completed
 [ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build,ami_build2

tests/logs/combined_push/test_combined_push/test_do_combined_push.txt

@@ -34,9 +34,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Marketplace VM push completed
 [ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build

tests/logs/combined_push/test_combined_push/test_do_combined_push_fail_one_workflow[CommunityVMPush].txt

@@ -34,9 +34,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Marketplace VM push completed
 [ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build

tests/logs/combined_push/test_combined_push/test_do_combined_push_overriden_destination.txt

@@ -34,9 +34,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Marketplace VM push completed
 [ INFO] Loading items from koji:https://fakekoji.com?vmi_build=ami_build

tests/logs/combined_push/test_combined_push/test_do_marketplace_push.txt

@@ -34,9 +34,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Collecting results
 [ INFO] Marketplace VM push completed

tests/logs/push/test_push/test_do_push.txt

@@ -57,13 +57,8 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan1 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan2 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA.
 [ INFO] Push already done for offer destination_offer_main on AZURE-NA.
 [ INFO] Push already done for offer destination_offer_main on AZURE-NA.

tests/logs/push/test_push/test_do_push_ami_correct_id.txt

@@ -35,12 +35,8 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to EMEA-DESTINATION on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to NA-DESTINATION on AWS-NA.
-[ DEBUG] Pushing ami_pushitem with image: fake-ami-id-for-na
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to NA-DESTINATION on AWS-NA.
 [ DEBUG] Pushing ami_pushitem with image: fake-ami-id-for-na
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to EMEA-DESTINATION on AWS-EMEA.
-[ DEBUG] Pushing ami_pushitem with image: fake-ami-id-for-emea
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to EMEA-DESTINATION on AWS-EMEA.
 [ DEBUG] Pushing ami_pushitem with image: fake-ami-id-for-emea
 [ INFO] Collecting results

tests/logs/push/test_push/test_do_push_azure_correct_sas.txt

@@ -21,12 +21,8 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to EMEA-DESTINATION on AZURE-EMEA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-EMEA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to NA-DESTINATION on AZURE-NA.
-[ DEBUG] Pushing vhd_pushitem with image: fake-azure-sas-for-na
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to NA-DESTINATION on AZURE-NA.
 [ DEBUG] Pushing vhd_pushitem with image: fake-azure-sas-for-na
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to EMEA-DESTINATION on AZURE-EMEA.
-[ DEBUG] Pushing vhd_pushitem with image: fake-azure-sas-for-emea
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to EMEA-DESTINATION on AZURE-EMEA.
 [ DEBUG] Pushing vhd_pushitem with image: fake-azure-sas-for-emea
 [ INFO] Collecting results

tests/logs/push/test_push/test_not_vmi_push_item.txt

@@ -36,9 +36,7 @@
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
 [ INFO] Preparing to publish the item ami_pushitem to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-EMEA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Collecting results
 [ INFO] Marketplace VM push completed

tests/logs/push/test_push/test_push_item_fail_publish.txt

@@ -57,16 +57,10 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ ERROR] Failed to publish ami_pushitem on AWS-NA: Random exception
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ ERROR] Failed to publish ami_pushitem on AWS-NA: Random exception
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
-[ ERROR] Failed to publish ami_pushitem on AWS-EMEA: Random exception
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ ERROR] Failed to publish ami_pushitem on AWS-EMEA: Random exception
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan1 on AZURE-NA.
-[ ERROR] Failed to publish vhd_pushitem on AZURE-NA: Random exception
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA.
 [ ERROR] Failed to publish vhd_pushitem on AZURE-NA: Random exception
 [ INFO] Collecting results

tests/logs/push/test_push/test_push_item_no_mapped_arch.txt

@@ -33,7 +33,6 @@
 [ INFO] Upload finished for ami_pushitem on AWS-NA
 [ INFO] Preparing to publish the item ami_pushitem to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Preparation complete for item ami_pushitem to AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Collecting results
 [ INFO] Marketplace VM push completed

tests/logs/push/test_push/test_push_item_rhcos_gov.txt

@@ -56,13 +56,8 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to ffffffff-ffff-ffff-ffff-ffffffffffff on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to 00000000-0000-0000-0000-000000000000 on AWS-EMEA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan1 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan2 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA.
 [ INFO] Push already done for offer destination_offer_main on AZURE-NA.
 [ INFO] Push already done for offer destination_offer_main on AZURE-NA.

tests/logs/push/test_push/test_push_item_wrong_arch.txt

@@ -24,9 +24,6 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan1 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan2 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to destination_offer_main/plan3 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to destination_offer_main/plan1 on AZURE-NA.
 [ INFO] Push already done for offer destination_offer_main on AZURE-NA.
 [ INFO] Push already done for offer destination_offer_main on AZURE-NA.

tests/logs/push/test_push/test_push_offline_starmap.txt

@@ -55,12 +55,8 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to new_azure_destination2 on AZURE-NA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to new_aws_na_destination on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to new_aws_na_destination on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to new_aws_emea_destination on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to new_aws_emea_destination on AWS-EMEA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to new_azure_destination1 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to new_azure_destination2 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to new_azure_destination1 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to new_azure_destination2 on AZURE-NA.
 [ INFO] Collecting results

tests/logs/push/test_push/test_push_overridden_destination.txt

@@ -55,12 +55,8 @@
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
 [ INFO] Preparing to publish the item vhd_pushitem to new_azure_destination2 on AZURE-NA.
 [ INFO] Preparation complete for item vhd_pushitem to AZURE-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to new_aws_na_destination on AWS-NA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to new_aws_na_destination on AWS-NA.
-[ INFO] Pushing "ami_pushitem" (pre-push=True) to new_aws_emea_destination on AWS-EMEA.
 [ INFO] Pushing "ami_pushitem" (pre-push=False) to new_aws_emea_destination on AWS-EMEA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to new_azure_destination1 on AZURE-NA.
-[ INFO] Pushing "vhd_pushitem" (pre-push=True) to new_azure_destination2 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to new_azure_destination1 on AZURE-NA.
 [ INFO] Pushing "vhd_pushitem" (pre-push=False) to new_azure_destination2 on AZURE-NA.
 [ INFO] Collecting results